From 5df60f47e86f76bc8fe6364ffe0693ec44132002 Mon Sep 17 00:00:00 2001
From: TennyZhuang <zty0826@gmail.com>
Date: Fri, 28 Jul 2023 00:28:19 +0800
Subject: add LinkedList::{retain,retain_mut}

Signed-off-by: TennyZhuang <zty0826@gmail.com>
---
 library/alloc/src/collections/linked_list.rs | 93 ++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/library/alloc/src/collections/linked_list.rs b/library/alloc/src/collections/linked_list.rs
index 052edf453f6..3bfbd1d9d2d 100644
--- a/library/alloc/src/collections/linked_list.rs
+++ b/library/alloc/src/collections/linked_list.rs
@@ -1024,6 +1024,99 @@ impl<T, A: Allocator> LinkedList<T, A> {
         }
     }
 
+    /// Retains only the elements specified by the predicate.
+    ///
+    /// In other words, remove all elements `e` for which `f(&e)` returns false.
+    /// This method operates in place, visiting each element exactly once in the
+    /// original order, and preserves the order of the retained elements.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(linked_list_retain)]
+    /// use std::collections::LinkedList;
+    ///
+    /// let mut d = LinkedList::new();
+    ///
+    /// d.push_front(1);
+    /// d.push_front(2);
+    /// d.push_front(3);
+    ///
+    /// d.retain(|&x| x % 2 == 0);
+    ///
+    /// assert_eq!(d.pop_front(), Some(2));
+    /// assert_eq!(d.pop_front(), None);
+    /// ```
+    ///
+    /// Because the elements are visited exactly once in the original order,
+    /// external state may be used to decide which elements to keep.
+    ///
+    /// ```
+    /// #![feature(linked_list_retain)]
+    /// use std::collections::LinkedList;
+    ///
+    /// let mut d = LinkedList::new();
+    ///
+    /// d.push_front(1);
+    /// d.push_front(2);
+    /// d.push_front(3);
+    ///
+    /// let keep = [false, true, false];
+    /// let mut iter = keep.iter();
+    /// d.retain(|_| *iter.next().unwrap());
+    /// assert_eq!(d.pop_front(), Some(2));
+    /// assert_eq!(d.pop_front(), None);
+    /// ```
+    #[unstable(feature = "linked_list_retain", issue = "114135")]
+    pub fn retain<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&T) -> bool,
+    {
+        self.retain_mut(|elem| f(elem));
+    }
+
+    /// Retains only the elements specified by the predicate.
+    ///
+    /// In other words, remove all elements `e` for which `f(&e)` returns false.
+    /// This method operates in place, visiting each element exactly once in the
+    /// original order, and preserves the order of the retained elements.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(linked_list_retain)]
+    /// use std::collections::LinkedList;
+    ///
+    /// let mut d = LinkedList::new();
+    ///
+    /// d.push_front(1);
+    /// d.push_front(2);
+    /// d.push_front(3);
+    ///
+    /// d.retain_mut(|x| if *x % 2 == 0 {
+    ///     *x += 1;
+    ///     true
+    /// } else {
+    ///     false
+    /// });
+    /// assert_eq!(d.pop_front(), Some(3));
+    /// assert_eq!(d.pop_front(), None);
+    /// ```
+    #[unstable(feature = "linked_list_retain", issue = "114135")]
+    pub fn retain_mut<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&mut T) -> bool,
+    {
+        let mut cursor = self.cursor_front_mut();
+        while let Some(node) = cursor.current() {
+            if !f(node) {
+                cursor.remove_current().unwrap();
+            } else {
+                cursor.move_next();
+            }
+        }
+    }
+
     /// Creates an iterator which uses a closure to determine if an element should be removed.
     ///
     /// If the closure returns true, then the element is removed and yielded.
-- 
cgit 1.4.1-3-g733a5


From 540921e468d770c47f460f1d0aeabbc8dae12ae9 Mon Sep 17 00:00:00 2001
From: Muhammad Hamza <muhammadhamza1311@gmail.com>
Date: Thu, 19 Oct 2023 22:33:09 +0500
Subject: Stablize arc_unwrap_or_clone

---
 library/alloc/src/rc.rs   | 3 +--
 library/alloc/src/sync.rs | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs
index dd7876bed76..b66f6fff798 100644
--- a/library/alloc/src/rc.rs
+++ b/library/alloc/src/rc.rs
@@ -1748,7 +1748,6 @@ impl<T: Clone, A: Allocator + Clone> Rc<T, A> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(arc_unwrap_or_clone)]
     /// # use std::{ptr, rc::Rc};
     /// let inner = String::from("test");
     /// let ptr = inner.as_ptr();
@@ -1769,7 +1768,7 @@ impl<T: Clone, A: Allocator + Clone> Rc<T, A> {
     /// assert!(ptr::eq(ptr, inner.as_ptr()));
     /// ```
     #[inline]
-    #[unstable(feature = "arc_unwrap_or_clone", issue = "93610")]
+    #[stable(feature = "arc_unwrap_or_clone", since = "CURRENT_RUSTC_VERSION")]
     pub fn unwrap_or_clone(this: Self) -> T {
         Rc::try_unwrap(this).unwrap_or_else(|rc| (*rc).clone())
     }
diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs
index 351e6c1a4b3..28ed311be16 100644
--- a/library/alloc/src/sync.rs
+++ b/library/alloc/src/sync.rs
@@ -2174,7 +2174,6 @@ impl<T: Clone, A: Allocator + Clone> Arc<T, A> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(arc_unwrap_or_clone)]
     /// # use std::{ptr, sync::Arc};
     /// let inner = String::from("test");
     /// let ptr = inner.as_ptr();
@@ -2195,7 +2194,7 @@ impl<T: Clone, A: Allocator + Clone> Arc<T, A> {
     /// assert!(ptr::eq(ptr, inner.as_ptr()));
     /// ```
     #[inline]
-    #[unstable(feature = "arc_unwrap_or_clone", issue = "93610")]
+    #[stable(feature = "arc_unwrap_or_clone", since = "CURRENT_RUSTC_VERSION")]
     pub fn unwrap_or_clone(this: Self) -> T {
         Arc::try_unwrap(this).unwrap_or_else(|arc| (*arc).clone())
     }
-- 
cgit 1.4.1-3-g733a5


From ad6dd6c624a2101d336203446bf6537bf44089f1 Mon Sep 17 00:00:00 2001
From: roblabla <unfiltered@roblab.la>
Date: Tue, 14 Nov 2023 17:45:04 +0100
Subject: Add new x86_64-win7-windows-msvc target

---
 compiler/rustc_target/src/spec/mod.rs              |  1 +
 .../src/spec/targets/x86_64_win7_windows_msvc.rs   | 18 +++++++++
 src/doc/rustc/src/SUMMARY.md                       |  1 +
 src/doc/rustc/src/platform-support.md              |  1 +
 .../src/platform-support/win7-windows-msvc.md      | 44 ++++++++++++++++++++++
 5 files changed, 65 insertions(+)
 create mode 100644 compiler/rustc_target/src/spec/targets/x86_64_win7_windows_msvc.rs
 create mode 100644 src/doc/rustc/src/platform-support/win7-windows-msvc.md

diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 9b0fedba092..295e81e5d13 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1604,6 +1604,7 @@ supported_targets! {
     ("aarch64-uwp-windows-msvc", aarch64_uwp_windows_msvc),
     ("x86_64-pc-windows-msvc", x86_64_pc_windows_msvc),
     ("x86_64-uwp-windows-msvc", x86_64_uwp_windows_msvc),
+    ("x86_64-win7-windows-msvc", x86_64_win7_windows_msvc),
     ("i686-pc-windows-msvc", i686_pc_windows_msvc),
     ("i686-uwp-windows-msvc", i686_uwp_windows_msvc),
     ("i586-pc-windows-msvc", i586_pc_windows_msvc),
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_win7_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/x86_64_win7_windows_msvc.rs
new file mode 100644
index 00000000000..5a59839ebc6
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/x86_64_win7_windows_msvc.rs
@@ -0,0 +1,18 @@
+use crate::spec::{base, Target};
+
+pub fn target() -> Target {
+    let mut base = base::windows_msvc::opts();
+    base.cpu = "x86-64".into();
+    base.plt_by_default = false;
+    base.max_atomic_width = Some(64);
+    base.vendor = "win7".into();
+
+    Target {
+        llvm_target: "x86_64-win7-windows-msvc".into(),
+        pointer_width: 64,
+        data_layout: "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+            .into(),
+        arch: "x86_64".into(),
+        options: base,
+    }
+}
diff --git a/src/doc/rustc/src/SUMMARY.md b/src/doc/rustc/src/SUMMARY.md
index 3bd90f7062d..4e08af3c3f8 100644
--- a/src/doc/rustc/src/SUMMARY.md
+++ b/src/doc/rustc/src/SUMMARY.md
@@ -55,6 +55,7 @@
     - [\*-unknown-uefi](platform-support/unknown-uefi.md)
     - [wasm32-wasi-preview1-threads](platform-support/wasm32-wasi-preview1-threads.md)
     - [wasm64-unknown-unknown](platform-support/wasm64-unknown-unknown.md)
+    - [\*-win7-windows-msvc](platform-support/win7-windows-msvc.md)
     - [x86_64-fortanix-unknown-sgx](platform-support/x86_64-fortanix-unknown-sgx.md)
     - [x86_64-unknown-none](platform-support/x86_64-unknown-none.md)
     - [x86_64h-apple-darwin](platform-support/x86_64h-apple-darwin.md)
diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index 3671fdd3fd2..2e1b5ea6799 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -351,6 +351,7 @@ target | std | host | notes
 [`x86_64-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | 64-bit OpenBSD
 `x86_64-uwp-windows-gnu` | ✓ |  |
 `x86_64-uwp-windows-msvc` | ✓ |  |
+[`x86_64-win7-windows-msvc`](platform-support/win7-windows-msvc.md) | ✓ |   | 64-bit Windows 7 support
 `x86_64-wrs-vxworks` | ? |  |
 [`x86_64h-apple-darwin`](platform-support/x86_64h-apple-darwin.md) | ✓ | ✓ | macOS with late-gen Intel (at least Haswell)
 
diff --git a/src/doc/rustc/src/platform-support/win7-windows-msvc.md b/src/doc/rustc/src/platform-support/win7-windows-msvc.md
new file mode 100644
index 00000000000..6fbf3b0223f
--- /dev/null
+++ b/src/doc/rustc/src/platform-support/win7-windows-msvc.md
@@ -0,0 +1,44 @@
+# *-win7-windows-msvc
+
+**Tier: 3**
+
+Windows targets continuing support of windows7.
+
+## Target maintainers
+
+- @roblabla
+
+## Requirements
+
+This target supports full the entirety of std. This is automatically tested
+every night on private infrastructure. Host tools may also work, though those
+are not currently tested.
+
+Those targets follow Windows calling convention for extern "C".
+
+Like with any other Windows target created binaries are in PE format.
+
+## Building the target
+
+You can build Rust with support for the targets by adding it to the target list in config.toml:
+
+```toml
+[build]
+build-stage = 1
+target = [ "x86_64-win7-windows-msvc" ]
+```
+
+## Building Rust programs
+
+Rust does not yet ship pre-compiled artifacts for this target. To compile for
+this target, you will either need to build Rust with the target enabled (see
+"Building the target" above), or build your own copy of `core` by using
+`build-std` or similar.
+
+## Testing
+
+Created binaries work fine on Windows or Wine using native hardware.
+
+## Cross-compilation toolchains and C code
+
+Compatible C code can be built with either MSVC's `cl.exe` or LLVM's clang-cl.
-- 
cgit 1.4.1-3-g733a5


From 85e73c1164952f3f4104b80e73d9f848d99b0e73 Mon Sep 17 00:00:00 2001
From: roblabla <unfiltered@roblab.la>
Date: Tue, 14 Nov 2023 17:44:03 +0100
Subject: Add i686-win7-windows-msvc target

---
 compiler/rustc_target/src/spec/mod.rs              |  1 +
 .../src/spec/targets/i686_win7_windows_msvc.rs     | 32 ++++++++++++++++++++++
 src/doc/rustc/src/platform-support.md              |  1 +
 3 files changed, 34 insertions(+)
 create mode 100644 compiler/rustc_target/src/spec/targets/i686_win7_windows_msvc.rs

diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 295e81e5d13..53251ed6a91 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1607,6 +1607,7 @@ supported_targets! {
     ("x86_64-win7-windows-msvc", x86_64_win7_windows_msvc),
     ("i686-pc-windows-msvc", i686_pc_windows_msvc),
     ("i686-uwp-windows-msvc", i686_uwp_windows_msvc),
+    ("i686-win7-windows-msvc", i686_win7_windows_msvc),
     ("i586-pc-windows-msvc", i586_pc_windows_msvc),
     ("thumbv7a-pc-windows-msvc", thumbv7a_pc_windows_msvc),
     ("thumbv7a-uwp-windows-msvc", thumbv7a_uwp_windows_msvc),
diff --git a/compiler/rustc_target/src/spec/targets/i686_win7_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/i686_win7_windows_msvc.rs
new file mode 100644
index 00000000000..ba80c23196e
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/i686_win7_windows_msvc.rs
@@ -0,0 +1,32 @@
+use crate::spec::{base, LinkerFlavor, Lld, Target};
+
+pub fn target() -> Target {
+    let mut base = base::windows_msvc::opts();
+    base.cpu = "pentium4".into();
+    base.max_atomic_width = Some(64);
+
+    base.add_pre_link_args(
+        LinkerFlavor::Msvc(Lld::No),
+        &[
+            // Mark all dynamic libraries and executables as compatible with the larger 4GiB address
+            // space available to x86 Windows binaries on x86_64.
+            "/LARGEADDRESSAWARE",
+            // Ensure the linker will only produce an image if it can also produce a table of
+            // the image's safe exception handlers.
+            // https://docs.microsoft.com/en-us/cpp/build/reference/safeseh-image-has-safe-exception-handlers
+            "/SAFESEH",
+        ],
+    );
+    // Workaround for #95429
+    base.has_thread_local = false;
+
+    Target {
+        llvm_target: "i686-pc-windows-msvc".into(),
+        pointer_width: 32,
+        data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
+            i64:64-f80:128-n8:16:32-a:0:32-S32"
+            .into(),
+        arch: "x86".into(),
+        options: base,
+    }
+}
diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index 2e1b5ea6799..289d09864cd 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -276,6 +276,7 @@ target | std | host | notes
 [`i686-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | 32-bit OpenBSD [^x86_32-floats-return-ABI]
 `i686-uwp-windows-gnu` | ? |  | [^x86_32-floats-return-ABI]
 `i686-uwp-windows-msvc` | ? |  | [^x86_32-floats-return-ABI]
+[`i686-win7-windows-msvc`](platform-support/win7-windows-msvc.md) | ✓ |   | 32-bit Windows 7 support [^x86_32-floats-return-ABI]
 `i686-wrs-vxworks` | ? |  | [^x86_32-floats-return-ABI]
 [`m68k-unknown-linux-gnu`](platform-support/m68k-unknown-linux-gnu.md) | ? |  | Motorola 680x0 Linux
 `mips-unknown-linux-gnu` | ✓ | ✓ | MIPS Linux (kernel 4.4, glibc 2.23)
-- 
cgit 1.4.1-3-g733a5


From e836561465da9977780626fd1a91f15bdf247985 Mon Sep 17 00:00:00 2001
From: roblabla <unfiltered@roblab.la>
Date: Tue, 21 Nov 2023 19:49:20 +0100
Subject: Improve -win7-windows-msvc documentation

---
 .../src/platform-support/win7-windows-msvc.md      | 54 ++++++++++++++++++----
 1 file changed, 45 insertions(+), 9 deletions(-)

diff --git a/src/doc/rustc/src/platform-support/win7-windows-msvc.md b/src/doc/rustc/src/platform-support/win7-windows-msvc.md
index 6fbf3b0223f..96613fb9be4 100644
--- a/src/doc/rustc/src/platform-support/win7-windows-msvc.md
+++ b/src/doc/rustc/src/platform-support/win7-windows-msvc.md
@@ -10,13 +10,13 @@ Windows targets continuing support of windows7.
 
 ## Requirements
 
-This target supports full the entirety of std. This is automatically tested
-every night on private infrastructure. Host tools may also work, though those
-are not currently tested.
+This target supports all of core, alloc, std and test. This is automatically
+tested every night on private infrastructure hosted by the maintainer. Host
+tools may also work, though those are not currently tested.
 
 Those targets follow Windows calling convention for extern "C".
 
-Like with any other Windows target created binaries are in PE format.
+Like any other Windows target, the created binaries are in PE format.
 
 ## Building the target
 
@@ -30,15 +30,51 @@ target = [ "x86_64-win7-windows-msvc" ]
 
 ## Building Rust programs
 
-Rust does not yet ship pre-compiled artifacts for this target. To compile for
-this target, you will either need to build Rust with the target enabled (see
-"Building the target" above), or build your own copy of `core` by using
-`build-std` or similar.
+Rust does not ship pre-compiled artifacts for this target. To compile for this
+target, you will either need to build Rust with the target enabled (see
+"Building the target" above), or build your own copy by using `build-std` or
+similar.
 
 ## Testing
 
-Created binaries work fine on Windows or Wine using native hardware.
+Created binaries work fine on Windows or Wine using native hardware. Remote
+testing is possible using the `remote-test-server` described [here](https://rustc-dev-guide.rust-lang.org/tests/running.html#running-tests-on-a-remote-machine).
 
 ## Cross-compilation toolchains and C code
 
 Compatible C code can be built with either MSVC's `cl.exe` or LLVM's clang-cl.
+
+Cross-compilation is possible using clang-cl/lld-link. It also requires the
+Windows SDK, which can be acquired using [`xwin`](https://github.com/Jake-Shadle/xwin).
+
+- Install `clang-cl` and `lld-link` on your machine, and make sure they are in
+  your $PATH.
+- Install `xwin`: `cargo install xwin`
+- Use `xwin` to install the Windows SDK: `xwin splat --output winsdk`
+- Create an `xwin-lld-link` script with the following content:
+
+  ```bash
+  #!/usr/bin/env bash
+  set -e
+  XWIN=/path/to/winsdk
+  lld-link "$@" /libpath:$XWIN/crt/lib/x86_64 /libpath:$XWIN/sdk/lib/um/x86_64 /libpath:$XWIN/sdk/lib/ucrt/x86_64
+  ```
+
+- Create an `xwin-clang-cl` script with the following content:
+
+  ```bash
+  #!/usr/bin/env bash
+  set -e
+  XWIN=/path/to/winsdk
+  clang-cl /imsvc "$XWIN/crt/include" /imsvc "$XWIN/sdk/include/ucrt" /imsvc "$XWIN/sdk/include/um" /imsvc "$XWIN/sdk/include/shared" --target="x86_64-pc-windows-msvc" "$@"
+  ```
+
+- In your config.toml, add the following lines:
+
+  ```toml
+  [target.x86_64-win7-windows-msvc]
+  linker = "path/to/xwin-lld-link"
+  cc = "path/to/xwin-clang-cl"
+  ```
+
+You should now be able to cross-compile the Rust std, and any rust program.
-- 
cgit 1.4.1-3-g733a5


From 39b495c3ff607cf8826f2d8800f1b9d774292977 Mon Sep 17 00:00:00 2001
From: Weihang Lo <me@weihanglo.tw>
Date: Sat, 25 Nov 2023 10:34:27 -0500
Subject: bootstrap: build `rustfix` doc from `src/tools/cargo`

---
 src/bootstrap/src/core/build_steps/doc.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
index 834f88dc891..c90a1b89759 100644
--- a/src/bootstrap/src/core/build_steps/doc.rs
+++ b/src/bootstrap/src/core/build_steps/doc.rs
@@ -1002,6 +1002,7 @@ tool_doc!(
         "cargo-test-macro",
         "cargo-test-support",
         "cargo-credential",
+        "rustfix",
         "mdman",
         // FIXME: this trips a license check in tidy.
         // "resolver-tests",
-- 
cgit 1.4.1-3-g733a5


From ee9a70dec066b5e40392eeb21b0a7b3282127ed0 Mon Sep 17 00:00:00 2001
From: Weihang Lo <me@weihanglo.tw>
Date: Sat, 25 Nov 2023 10:34:58 -0500
Subject: bootstrap: sort items of cargo docs

---
 src/bootstrap/src/core/build_steps/doc.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
index c90a1b89759..acddd4d77fa 100644
--- a/src/bootstrap/src/core/build_steps/doc.rs
+++ b/src/bootstrap/src/core/build_steps/doc.rs
@@ -996,14 +996,14 @@ tool_doc!(
     in_tree = false,
     crates = [
         "cargo",
+        "cargo-credential",
         "cargo-platform",
-        "cargo-util",
-        "crates-io",
         "cargo-test-macro",
         "cargo-test-support",
-        "cargo-credential",
-        "rustfix",
+        "cargo-util",
+        "crates-io",
         "mdman",
+        "rustfix",
         // FIXME: this trips a license check in tidy.
         // "resolver-tests",
     ]
-- 
cgit 1.4.1-3-g733a5


From 72321011cab6d3bb338e5776bb3bf293c7699f42 Mon Sep 17 00:00:00 2001
From: Weihang Lo <me@weihanglo.tw>
Date: Sat, 25 Nov 2023 10:35:43 -0500
Subject: bootstrap: remove unused comment

resolver-tests is for test only so no need to document it.
---
 src/bootstrap/src/core/build_steps/doc.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
index acddd4d77fa..cf3f5bc983e 100644
--- a/src/bootstrap/src/core/build_steps/doc.rs
+++ b/src/bootstrap/src/core/build_steps/doc.rs
@@ -1004,8 +1004,6 @@ tool_doc!(
         "crates-io",
         "mdman",
         "rustfix",
-        // FIXME: this trips a license check in tidy.
-        // "resolver-tests",
     ]
 );
 tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, crates = ["tidy"]);
-- 
cgit 1.4.1-3-g733a5


From e4cfe038adffb0cf5785d7c95f809364c6a39f5f Mon Sep 17 00:00:00 2001
From: Young-Flash <871946895@qq.com>
Date: Fri, 1 Dec 2023 22:17:31 +0800
Subject: fix: correct the arg for 'suggest to use associated function syntax'
 diagnostic

---
 compiler/rustc_hir_typeck/src/method/suggest.rs    |  8 +++--
 .../suggest-assoc-fn-call-without-receiver.fixed   | 16 +++++++++
 .../suggest-assoc-fn-call-without-receiver.rs      | 16 +++++++++
 .../suggest-assoc-fn-call-without-receiver.stderr  | 41 ++++++++++++++++++++++
 4 files changed, 78 insertions(+), 3 deletions(-)
 create mode 100644 tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed
 create mode 100644 tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs
 create mode 100644 tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr

diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index a7764f4ff96..117b60cf42e 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -1591,10 +1591,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             {
                 let sig = self.tcx.fn_sig(assoc.def_id).instantiate_identity();
                 sig.inputs().skip_binder().get(0).and_then(|first| {
-                    if first.peel_refs() == rcvr_ty.peel_refs() {
-                        None
-                    } else {
+                    let impl_ty = self.tcx.type_of(*impl_did).instantiate_identity();
+                    // if the type of first arg is the same as the current impl type, we should take the first arg into assoc function
+                    if first.peel_refs() == impl_ty {
                         Some(first.ref_mutability().map_or("", |mutbl| mutbl.ref_prefix_str()))
+                    } else {
+                        None
                     }
                 })
             } else {
diff --git a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed
new file mode 100644
index 00000000000..6e679134d63
--- /dev/null
+++ b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed
@@ -0,0 +1,16 @@
+// run-rustfix
+
+struct A {}
+
+impl A {
+    fn hello(_a: i32) {}
+    fn test(_a: Self, _b: i32) {}
+}
+
+fn main() {
+    let _a = A {};
+    A::hello(1);
+    //~^ ERROR no method named `hello` found
+    A::test(_a, 1);
+    //~^ ERROR no method named `test` found
+}
diff --git a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs
new file mode 100644
index 00000000000..67c2cc1bed5
--- /dev/null
+++ b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs
@@ -0,0 +1,16 @@
+// run-rustfix
+
+struct A {}
+
+impl A {
+    fn hello(_a: i32) {}
+    fn test(_a: Self, _b: i32) {}
+}
+
+fn main() {
+    let _a = A {};
+    _a.hello(1);
+    //~^ ERROR no method named `hello` found
+    _a.test(1);
+    //~^ ERROR no method named `test` found
+}
diff --git a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr
new file mode 100644
index 00000000000..ed227cbab39
--- /dev/null
+++ b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr
@@ -0,0 +1,41 @@
+error[E0599]: no method named `hello` found for struct `A` in the current scope
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:12:8
+   |
+LL | struct A {}
+   | -------- method `hello` not found for this struct
+...
+LL |     _a.hello(1);
+   |     ---^^^^^---
+   |     |  |
+   |     |  this is an associated function, not a method
+   |     help: use associated function syntax instead: `A::hello(1)`
+   |
+   = note: found the following associated functions; to be used as methods, functions must have a `self` parameter
+note: the candidate is defined in an impl for the type `A`
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:6:5
+   |
+LL |     fn hello(_a: i32) {}
+   |     ^^^^^^^^^^^^^^^^^
+
+error[E0599]: no method named `test` found for struct `A` in the current scope
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:14:8
+   |
+LL | struct A {}
+   | -------- method `test` not found for this struct
+...
+LL |     _a.test(1);
+   |     ---^^^^---
+   |     |  |
+   |     |  this is an associated function, not a method
+   |     help: use associated function syntax instead: `A::test(_a, 1)`
+   |
+   = note: found the following associated functions; to be used as methods, functions must have a `self` parameter
+note: the candidate is defined in an impl for the type `A`
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:7:5
+   |
+LL |     fn test(_a: Self, _b: i32) {}
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0599`.
-- 
cgit 1.4.1-3-g733a5


From caa488b96e65131e4d70f219d5e89008031f9229 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 27 Nov 2023 01:08:39 +0100
Subject: Add tests

---
 tests/ui/never_patterns/check.rs              |  30 +++++++
 tests/ui/never_patterns/check.stderr          |   8 ++
 tests/ui/parser/match-arm-without-body.rs     |  68 ++++++++++++++
 tests/ui/parser/match-arm-without-body.stderr | 122 ++++++++++++++++++++++++++
 tests/ui/pattern/never_patterns.rs            |   2 +-
 5 files changed, 229 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/never_patterns/check.rs
 create mode 100644 tests/ui/never_patterns/check.stderr
 create mode 100644 tests/ui/parser/match-arm-without-body.rs
 create mode 100644 tests/ui/parser/match-arm-without-body.stderr

diff --git a/tests/ui/never_patterns/check.rs b/tests/ui/never_patterns/check.rs
new file mode 100644
index 00000000000..bcc3a760c10
--- /dev/null
+++ b/tests/ui/never_patterns/check.rs
@@ -0,0 +1,30 @@
+#![feature(never_patterns)]
+#![allow(incomplete_features)]
+
+enum Void {}
+
+fn main() {}
+
+macro_rules! never {
+    () => { ! }
+}
+
+fn no_arms_or_guards(x: Void) {
+    match None::<Void> {
+        Some(!) => {}
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true,
+        //~^ ERROR expected one of
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true => {}
+        None => {}
+    }
+    match None::<Void> {
+        Some(never!()) => {},
+        None => {}
+    }
+}
diff --git a/tests/ui/never_patterns/check.stderr b/tests/ui/never_patterns/check.stderr
new file mode 100644
index 00000000000..d7cdd64840f
--- /dev/null
+++ b/tests/ui/never_patterns/check.stderr
@@ -0,0 +1,8 @@
+error: expected one of `.`, `=>`, `?`, or an operator, found `,`
+  --> $DIR/check.rs:18:24
+   |
+LL |         Some(!) if true,
+   |                        ^ expected one of `.`, `=>`, `?`, or an operator
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/parser/match-arm-without-body.rs b/tests/ui/parser/match-arm-without-body.rs
new file mode 100644
index 00000000000..5f009c7a355
--- /dev/null
+++ b/tests/ui/parser/match-arm-without-body.rs
@@ -0,0 +1,68 @@
+macro_rules! pat {
+    () => { Some(_) }
+}
+
+fn main() {
+    match Some(false) {
+        Some(_)
+    }
+    //~^ ERROR expected one of
+    match Some(false) {
+        Some(_)
+        _ => {}
+        //~^ ERROR expected one of
+    }
+    match Some(false) {
+        Some(_),
+        //~^ ERROR unexpected `,` in pattern
+        //~| HELP try adding parentheses to match on a tuple
+        //~| HELP or a vertical bar to match on multiple alternatives
+    }
+    match Some(false) {
+        Some(_),
+        //~^ ERROR unexpected `,` in pattern
+        //~| HELP try adding parentheses to match on a tuple
+        //~| HELP or a vertical bar to match on multiple alternatives
+        _ => {}
+    }
+    match Some(false) {
+        Some(_) if true
+    }
+    //~^ ERROR expected one of
+    match Some(false) {
+        Some(_) if true
+        _ => {}
+        //~^ ERROR expected one of
+    }
+    match Some(false) {
+        Some(_) if true,
+        //~^ ERROR expected one of
+    }
+    match Some(false) {
+        Some(_) if true,
+        //~^ ERROR expected one of
+        _ => {}
+    }
+    match Some(false) {
+        pat!()
+    }
+    //~^ ERROR expected one of
+    match Some(false) {
+        pat!(),
+        //~^ ERROR unexpected `,` in pattern
+    }
+    match Some(false) {
+        pat!() if true,
+        //~^ ERROR expected one of
+    }
+    match Some(false) {
+        pat!()
+        _ => {}
+        //~^ ERROR expected one of
+    }
+    match Some(false) {
+        pat!(),
+        //~^ ERROR unexpected `,` in pattern
+        _ => {}
+    }
+}
diff --git a/tests/ui/parser/match-arm-without-body.stderr b/tests/ui/parser/match-arm-without-body.stderr
new file mode 100644
index 00000000000..210007628db
--- /dev/null
+++ b/tests/ui/parser/match-arm-without-body.stderr
@@ -0,0 +1,122 @@
+error: expected one of `=>`, `if`, or `|`, found `}`
+  --> $DIR/match-arm-without-body.rs:8:5
+   |
+LL |         Some(_)
+   |                - expected one of `=>`, `if`, or `|`
+LL |     }
+   |     ^ unexpected token
+
+error: expected one of `=>`, `if`, or `|`, found reserved identifier `_`
+  --> $DIR/match-arm-without-body.rs:12:9
+   |
+LL |         Some(_)
+   |                - expected one of `=>`, `if`, or `|`
+LL |         _ => {}
+   |         ^ unexpected token
+
+error: unexpected `,` in pattern
+  --> $DIR/match-arm-without-body.rs:16:16
+   |
+LL |         Some(_),
+   |                ^
+   |
+help: try adding parentheses to match on a tuple...
+   |
+LL |         (Some(_),)
+   |         +        +
+help: ...or a vertical bar to match on multiple alternatives
+   |
+LL |         Some(_) |
+   |
+
+error: unexpected `,` in pattern
+  --> $DIR/match-arm-without-body.rs:22:16
+   |
+LL |         Some(_),
+   |                ^
+   |
+help: try adding parentheses to match on a tuple...
+   |
+LL ~         (Some(_),
+LL |
+LL |
+LL |
+LL ~         _) => {}
+   |
+help: ...or a vertical bar to match on multiple alternatives
+   |
+LL ~         Some(_) |
+LL +
+LL +
+LL +
+LL ~         _ => {}
+   |
+
+error: expected one of `.`, `=>`, `?`, or an operator, found `}`
+  --> $DIR/match-arm-without-body.rs:30:5
+   |
+LL |         Some(_) if true
+   |                        - expected one of `.`, `=>`, `?`, or an operator
+LL |     }
+   |     ^ unexpected token
+
+error: expected one of `.`, `=>`, `?`, or an operator, found reserved identifier `_`
+  --> $DIR/match-arm-without-body.rs:34:9
+   |
+LL |         Some(_) if true
+   |                        - expected one of `.`, `=>`, `?`, or an operator
+LL |         _ => {}
+   |         ^ unexpected token
+
+error: expected one of `.`, `=>`, `?`, or an operator, found `,`
+  --> $DIR/match-arm-without-body.rs:38:24
+   |
+LL |         Some(_) if true,
+   |                        ^ expected one of `.`, `=>`, `?`, or an operator
+
+error: expected one of `.`, `=>`, `?`, or an operator, found `,`
+  --> $DIR/match-arm-without-body.rs:42:24
+   |
+LL |         Some(_) if true,
+   |                        ^ expected one of `.`, `=>`, `?`, or an operator
+
+error: expected one of `=>`, `if`, or `|`, found `}`
+  --> $DIR/match-arm-without-body.rs:48:5
+   |
+LL |         pat!()
+   |               - expected one of `=>`, `if`, or `|`
+LL |     }
+   |     ^ unexpected token
+
+error: unexpected `,` in pattern
+  --> $DIR/match-arm-without-body.rs:51:15
+   |
+LL |         pat!(),
+   |               ^
+   |
+   = note: macros cannot expand to match arms
+
+error: expected one of `.`, `=>`, `?`, or an operator, found `,`
+  --> $DIR/match-arm-without-body.rs:55:23
+   |
+LL |         pat!() if true,
+   |                       ^ expected one of `.`, `=>`, `?`, or an operator
+
+error: expected one of `=>`, `if`, or `|`, found reserved identifier `_`
+  --> $DIR/match-arm-without-body.rs:60:9
+   |
+LL |         pat!()
+   |               - expected one of `=>`, `if`, or `|`
+LL |         _ => {}
+   |         ^ unexpected token
+
+error: unexpected `,` in pattern
+  --> $DIR/match-arm-without-body.rs:64:15
+   |
+LL |         pat!(),
+   |               ^
+   |
+   = note: macros cannot expand to match arms
+
+error: aborting due to 13 previous errors
+
diff --git a/tests/ui/pattern/never_patterns.rs b/tests/ui/pattern/never_patterns.rs
index e2e17e0e9a7..fdba1b8e087 100644
--- a/tests/ui/pattern/never_patterns.rs
+++ b/tests/ui/pattern/never_patterns.rs
@@ -84,7 +84,7 @@ fn never_and_bindings() {
         //~^ ERROR: is not bound in all patterns
     }
     let (Ok(_x) | Err(&!)) = x;
-        //~^ ERROR: is not bound in all patterns
+    //~^ ERROR: is not bound in all patterns
 
     // FIXME(never_patterns): A never pattern mustn't have bindings.
     match x {
-- 
cgit 1.4.1-3-g733a5


From 378abbc604897945cf86a9eb0c16371ebd571b54 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:16:25 +0000
Subject: FileCheck address_of_pair.

---
 tests/mir-opt/const_prop/address_of_pair.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/address_of_pair.rs b/tests/mir-opt/const_prop/address_of_pair.rs
index 169469a0739..730ebe2ca63 100644
--- a/tests/mir-opt/const_prop/address_of_pair.rs
+++ b/tests/mir-opt/const_prop/address_of_pair.rs
@@ -1,8 +1,21 @@
-// skip-filecheck
 // unit-test: ConstProp
 
 // EMIT_MIR address_of_pair.fn0.ConstProp.diff
 pub fn fn0() -> bool {
+    // CHECK-LABEL: fn fn0(
+    // CHECK: debug pair => [[pair:_.*]];
+    // CHECK: debug ptr => [[ptr:_.*]];
+    // CHECK: debug ret => [[ret:_.*]];
+    // CHECK: (*[[ptr]]) = const true;
+    // CHECK-NOT: = const false;
+    // CHECK-NOT: = const true;
+    // CHECK: [[tmp:_.*]] = ([[pair]].1: bool);
+    // CHECK-NOT: = const false;
+    // CHECK-NOT: = const true;
+    // CHECK: [[ret]] = Not(move [[tmp]]);
+    // CHECK-NOT: = const false;
+    // CHECK-NOT: = const true;
+    // CHECK: _0 = [[ret]];
     let mut pair = (1, false);
     let ptr = core::ptr::addr_of_mut!(pair.1);
     pair = (1, false);
-- 
cgit 1.4.1-3-g733a5


From 6564bac53254ae68fd5788bb0ce4b83f9ba4f8e4 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:17:04 +0000
Subject: FileCheck aggregate.

---
 .../aggregate.foo.PreCodegen.after.panic-abort.mir | 49 ----------------------
 ...aggregate.foo.PreCodegen.after.panic-unwind.mir | 49 ----------------------
 ...aggregate.main.PreCodegen.after.panic-abort.mir | 36 ----------------
 ...ggregate.main.PreCodegen.after.panic-unwind.mir | 36 ----------------
 tests/mir-opt/const_prop/aggregate.rs              | 18 ++++++--
 5 files changed, 14 insertions(+), 174 deletions(-)
 delete mode 100644 tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-abort.mir
 delete mode 100644 tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-unwind.mir
 delete mode 100644 tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-abort.mir
 delete mode 100644 tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-unwind.mir

diff --git a/tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-abort.mir b/tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-abort.mir
deleted file mode 100644
index b9c5859cade..00000000000
--- a/tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-abort.mir
+++ /dev/null
@@ -1,49 +0,0 @@
-// MIR for `foo` after PreCodegen
-
-fn foo(_1: u8) -> () {
-    debug x => _1;
-    let mut _0: ();
-    let _2: i32;
-    let mut _3: i32;
-    let mut _4: (i32, u8);
-    let mut _5: u8;
-    let mut _7: i32;
-    let mut _8: (u8, i32);
-    let mut _9: u8;
-    scope 1 {
-        debug first => _2;
-        let _6: i32;
-        scope 2 {
-            debug second => _6;
-        }
-    }
-
-    bb0: {
-        StorageLive(_2);
-        StorageLive(_3);
-        StorageLive(_4);
-        StorageLive(_5);
-        _5 = _1;
-        _4 = (const 0_i32, move _5);
-        StorageDead(_5);
-        _3 = const 0_i32;
-        _2 = const 1_i32;
-        StorageDead(_3);
-        StorageDead(_4);
-        StorageLive(_6);
-        StorageLive(_7);
-        StorageLive(_8);
-        StorageLive(_9);
-        _9 = _1;
-        _8 = (move _9, const 1_i32);
-        StorageDead(_9);
-        _7 = const 1_i32;
-        _6 = const 3_i32;
-        StorageDead(_7);
-        StorageDead(_8);
-        _0 = const ();
-        StorageDead(_6);
-        StorageDead(_2);
-        return;
-    }
-}
diff --git a/tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-unwind.mir
deleted file mode 100644
index b9c5859cade..00000000000
--- a/tests/mir-opt/const_prop/aggregate.foo.PreCodegen.after.panic-unwind.mir
+++ /dev/null
@@ -1,49 +0,0 @@
-// MIR for `foo` after PreCodegen
-
-fn foo(_1: u8) -> () {
-    debug x => _1;
-    let mut _0: ();
-    let _2: i32;
-    let mut _3: i32;
-    let mut _4: (i32, u8);
-    let mut _5: u8;
-    let mut _7: i32;
-    let mut _8: (u8, i32);
-    let mut _9: u8;
-    scope 1 {
-        debug first => _2;
-        let _6: i32;
-        scope 2 {
-            debug second => _6;
-        }
-    }
-
-    bb0: {
-        StorageLive(_2);
-        StorageLive(_3);
-        StorageLive(_4);
-        StorageLive(_5);
-        _5 = _1;
-        _4 = (const 0_i32, move _5);
-        StorageDead(_5);
-        _3 = const 0_i32;
-        _2 = const 1_i32;
-        StorageDead(_3);
-        StorageDead(_4);
-        StorageLive(_6);
-        StorageLive(_7);
-        StorageLive(_8);
-        StorageLive(_9);
-        _9 = _1;
-        _8 = (move _9, const 1_i32);
-        StorageDead(_9);
-        _7 = const 1_i32;
-        _6 = const 3_i32;
-        StorageDead(_7);
-        StorageDead(_8);
-        _0 = const ();
-        StorageDead(_6);
-        StorageDead(_2);
-        return;
-    }
-}
diff --git a/tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-abort.mir b/tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-abort.mir
deleted file mode 100644
index 44a85a5636b..00000000000
--- a/tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-abort.mir
+++ /dev/null
@@ -1,36 +0,0 @@
-// MIR for `main` after PreCodegen
-
-fn main() -> () {
-    let mut _0: ();
-    let _1: u8;
-    let mut _2: u8;
-    let mut _3: (i32, u8, i32);
-    let _4: ();
-    let mut _5: u8;
-    scope 1 {
-        debug x => _1;
-    }
-
-    bb0: {
-        StorageLive(_1);
-        StorageLive(_2);
-        StorageLive(_3);
-        _3 = (const 0_i32, const 1_u8, const 2_i32);
-        _2 = const 1_u8;
-        _1 = const 1_u8;
-        StorageDead(_2);
-        StorageDead(_3);
-        StorageLive(_4);
-        StorageLive(_5);
-        _5 = const 1_u8;
-        _4 = foo(const 1_u8) -> [return: bb1, unwind unreachable];
-    }
-
-    bb1: {
-        StorageDead(_5);
-        StorageDead(_4);
-        _0 = const ();
-        StorageDead(_1);
-        return;
-    }
-}
diff --git a/tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-unwind.mir
deleted file mode 100644
index 2c7bdbb5055..00000000000
--- a/tests/mir-opt/const_prop/aggregate.main.PreCodegen.after.panic-unwind.mir
+++ /dev/null
@@ -1,36 +0,0 @@
-// MIR for `main` after PreCodegen
-
-fn main() -> () {
-    let mut _0: ();
-    let _1: u8;
-    let mut _2: u8;
-    let mut _3: (i32, u8, i32);
-    let _4: ();
-    let mut _5: u8;
-    scope 1 {
-        debug x => _1;
-    }
-
-    bb0: {
-        StorageLive(_1);
-        StorageLive(_2);
-        StorageLive(_3);
-        _3 = (const 0_i32, const 1_u8, const 2_i32);
-        _2 = const 1_u8;
-        _1 = const 1_u8;
-        StorageDead(_2);
-        StorageDead(_3);
-        StorageLive(_4);
-        StorageLive(_5);
-        _5 = const 1_u8;
-        _4 = foo(const 1_u8) -> [return: bb1, unwind continue];
-    }
-
-    bb1: {
-        StorageDead(_5);
-        StorageDead(_4);
-        _0 = const ();
-        StorageDead(_1);
-        return;
-    }
-}
diff --git a/tests/mir-opt/const_prop/aggregate.rs b/tests/mir-opt/const_prop/aggregate.rs
index 2e043af08bf..fa716b0843d 100644
--- a/tests/mir-opt/const_prop/aggregate.rs
+++ b/tests/mir-opt/const_prop/aggregate.rs
@@ -1,19 +1,29 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
 // compile-flags: -O
 
 // EMIT_MIR aggregate.main.ConstProp.diff
-// EMIT_MIR aggregate.main.PreCodegen.after.mir
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK-NOT: = Add(
+    // CHECK: [[x]] = const 1_u8;
+    // CHECK-NOT: = Add(
+    // CHECK: foo(const 1_u8)
     let x = (0, 1, 2).1 + 0;
     foo(x);
 }
 
+// Verify that we still propagate if part of the aggregate is not known.
 // EMIT_MIR aggregate.foo.ConstProp.diff
-// EMIT_MIR aggregate.foo.PreCodegen.after.mir
 fn foo(x: u8) {
-    // Verify that we still propagate if part of the aggregate is not known.
+    // CHECK-LABEL: fn foo(
+    // CHECK: debug first => [[first:_.*]];
+    // CHECK: debug second => [[second:_.*]];
+    // CHECK-NOT: = Add(
+    // CHECK: [[first]] = const 1_i32;
+    // CHECK-NOT: = Add(
+    // CHECK: [[second]] = const 3_i32;
     let first = (0, x).0 + 1;
     let second = (x, 1).1 + 2;
 }
-- 
cgit 1.4.1-3-g733a5


From 9f01d9d1b61cd013bc8c21711f1f27455bffc21a Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:17:20 +0000
Subject: FileCheck array_index.

---
 tests/mir-opt/const_prop/array_index.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/mir-opt/const_prop/array_index.rs b/tests/mir-opt/const_prop/array_index.rs
index 3bd2321653d..c4c46d78f75 100644
--- a/tests/mir-opt/const_prop/array_index.rs
+++ b/tests/mir-opt/const_prop/array_index.rs
@@ -1,9 +1,11 @@
-// skip-filecheck
-// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
 
 // EMIT_MIR array_index.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: [[x]] = const 2_u32;
     let x: u32 = [0, 1, 2, 3][2];
 }
-- 
cgit 1.4.1-3-g733a5


From 0d5bc872a9715501f13f6d848ce1eddc073c33ac Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:17:37 +0000
Subject: FileCheck bad_op_div_by_zero.

---
 tests/mir-opt/const_prop/bad_op_div_by_zero.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/mir-opt/const_prop/bad_op_div_by_zero.rs b/tests/mir-opt/const_prop/bad_op_div_by_zero.rs
index ab41f64a573..0e8765a0771 100644
--- a/tests/mir-opt/const_prop/bad_op_div_by_zero.rs
+++ b/tests/mir-opt/const_prop/bad_op_div_by_zero.rs
@@ -1,9 +1,15 @@
-// skip-filecheck
-// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
 // EMIT_MIR bad_op_div_by_zero.main.ConstProp.diff
 #[allow(unconditional_panic)]
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: debug _z => [[z:_.*]];
+    // CHECK: assert(!const true, "attempt to divide `{}` by zero", const 1_i32)
+    // CHECK: assert(!const false, "attempt to compute `{} / {}`, which would overflow", const 1_i32, const 0_i32)
+    // CHECK: [[z]] = Div(const 1_i32, const 0_i32);
     let y = 0;
     let _z = 1 / y;
 }
-- 
cgit 1.4.1-3-g733a5


From 97f03cb8987cc115746845f5d3eab9ee8e7523d1 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:17:49 +0000
Subject: FileCheck bad_op_mod_by_zero.

---
 tests/mir-opt/const_prop/bad_op_mod_by_zero.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/bad_op_mod_by_zero.rs b/tests/mir-opt/const_prop/bad_op_mod_by_zero.rs
index e747b21cf9b..d895d9e2155 100644
--- a/tests/mir-opt/const_prop/bad_op_mod_by_zero.rs
+++ b/tests/mir-opt/const_prop/bad_op_mod_by_zero.rs
@@ -1,9 +1,16 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
 // EMIT_MIR bad_op_mod_by_zero.main.ConstProp.diff
 #[allow(unconditional_panic)]
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: debug _z => [[z:_.*]];
+    // CHECK: assert(!const true, "attempt to calculate the remainder of `{}` with a divisor of
+    // zero", const 1_i32)
+    // CHECK: assert(!const false, "attempt to compute the remainder of `{} % {}`, which would overflow", const 1_i32, const 0_i32)
+    // CHECK: [[z]] = Rem(const 1_i32, const 0_i32);
     let y = 0;
     let _z = 1 % y;
 }
-- 
cgit 1.4.1-3-g733a5


From e8e35c81271a78c59336b54dcd3b65ae13122bf4 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:18:02 +0000
Subject: FileCheck bad_op_unsafe_oob_for_slices.

---
 tests/mir-opt/const_prop/bad_op_unsafe_oob_for_slices.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/bad_op_unsafe_oob_for_slices.rs b/tests/mir-opt/const_prop/bad_op_unsafe_oob_for_slices.rs
index 38c97a4cf0e..266105c11f2 100644
--- a/tests/mir-opt/const_prop/bad_op_unsafe_oob_for_slices.rs
+++ b/tests/mir-opt/const_prop/bad_op_unsafe_oob_for_slices.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
@@ -6,6 +5,10 @@
 // EMIT_MIR bad_op_unsafe_oob_for_slices.main.ConstProp.diff
 #[allow(unconditional_panic)]
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: debug _b => [[b:_.*]];
+    // CHECK: [[b]] = (*[[a]])[3 of 4];
     let a: *const [_] = &[1, 2, 3];
     unsafe {
         let _b = (*a)[3];
-- 
cgit 1.4.1-3-g733a5


From 3fc03948a8470fee27d5eee02ad6f4b29f6977d8 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:18:14 +0000
Subject: FileCheck boolean_identities.

---
 tests/mir-opt/const_prop/boolean_identities.rs     | 12 ++++++--
 .../boolean_identities.test.ConstProp.diff         | 32 ++++++++++++++++++----
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/tests/mir-opt/const_prop/boolean_identities.rs b/tests/mir-opt/const_prop/boolean_identities.rs
index 781cce8c7dd..2aa038034d8 100644
--- a/tests/mir-opt/const_prop/boolean_identities.rs
+++ b/tests/mir-opt/const_prop/boolean_identities.rs
@@ -1,10 +1,16 @@
-// skip-filecheck
 // unit-test: ConstProp
-// compile-flags: -O -Zmir-opt-level=4
 
 // EMIT_MIR boolean_identities.test.ConstProp.diff
 pub fn test(x: bool, y: bool) -> bool {
-    (y | true) & (x & false)
+    // CHECK-LABEL: fn test(
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: debug b => [[b:_.*]];
+    // CHECK: [[a]] = const true;
+    // CHECK: [[b]] = const false;
+    // CHECK: _0 = const false;
+    let a = (y | true);
+    let b = (x & false);
+    a & b
 }
 
 fn main() {
diff --git a/tests/mir-opt/const_prop/boolean_identities.test.ConstProp.diff b/tests/mir-opt/const_prop/boolean_identities.test.ConstProp.diff
index d805341991d..41e1acdff59 100644
--- a/tests/mir-opt/const_prop/boolean_identities.test.ConstProp.diff
+++ b/tests/mir-opt/const_prop/boolean_identities.test.ConstProp.diff
@@ -5,20 +5,42 @@
       debug x => _1;
       debug y => _2;
       let mut _0: bool;
-      let mut _3: bool;
+      let _3: bool;
       let mut _4: bool;
-      let mut _5: bool;
       let mut _6: bool;
+      let mut _7: bool;
+      let mut _8: bool;
+      scope 1 {
+          debug a => _3;
+          let _5: bool;
+          scope 2 {
+              debug b => _5;
+          }
+      }
   
       bb0: {
           StorageLive(_3);
--         _3 = BitOr(_2, const true);
+          StorageLive(_4);
+          _4 = _2;
+-         _3 = BitOr(move _4, const true);
 +         _3 = const true;
+          StorageDead(_4);
           StorageLive(_5);
--         _5 = BitAnd(_1, const false);
--         _0 = BitAnd(move _3, move _5);
+          StorageLive(_6);
+          _6 = _1;
+-         _5 = BitAnd(move _6, const false);
 +         _5 = const false;
+          StorageDead(_6);
+          StorageLive(_7);
+-         _7 = _3;
++         _7 = const true;
+          StorageLive(_8);
+-         _8 = _5;
+-         _0 = BitAnd(move _7, move _8);
++         _8 = const false;
 +         _0 = const false;
+          StorageDead(_8);
+          StorageDead(_7);
           StorageDead(_5);
           StorageDead(_3);
           return;
-- 
cgit 1.4.1-3-g733a5


From b8f2f639310193aa2de2ffc1f5685f154dcd57cf Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:18:21 +0000
Subject: FileCheck boxes.

---
 tests/mir-opt/const_prop/boxes.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/boxes.rs b/tests/mir-opt/const_prop/boxes.rs
index c6807ece199..90a8e33e823 100644
--- a/tests/mir-opt/const_prop/boxes.rs
+++ b/tests/mir-opt/const_prop/boxes.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // unit-test: ConstProp
 // compile-flags: -O
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
@@ -9,6 +8,11 @@
 
 // EMIT_MIR boxes.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: (*{{_.*}}) = const 42_i32;
+    // CHECK: [[tmp:_.*]] = (*{{_.*}});
+    // CHECK: [[x]] = Add(move [[tmp]], const 0_i32);
     let x = *(#[rustc_box]
     Box::new(42))
         + 0;
-- 
cgit 1.4.1-3-g733a5


From e6a1b77cd106a5a2a1571e74039fc466559ff8db Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:18:47 +0000
Subject: FileCheck cast.

---
 tests/mir-opt/const_prop/cast.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/mir-opt/const_prop/cast.rs b/tests/mir-opt/const_prop/cast.rs
index 3d543badace..b81c2740a73 100644
--- a/tests/mir-opt/const_prop/cast.rs
+++ b/tests/mir-opt/const_prop/cast.rs
@@ -1,9 +1,12 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR cast.main.ConstProp.diff
 
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: [[x]] = const 42_u32;
+    // CHECK: [[y]] = const 42_u8;
     let x = 42u8 as u32;
-
     let y = 42u32 as u8;
 }
-- 
cgit 1.4.1-3-g733a5


From 7f328d2a4462c3a8afcaf773a22c2b9c300fef12 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:18:55 +0000
Subject: FileCheck checked_add.

---
 tests/mir-opt/const_prop/checked_add.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/checked_add.rs b/tests/mir-opt/const_prop/checked_add.rs
index 6a53aced091..571a5cc4e4d 100644
--- a/tests/mir-opt/const_prop/checked_add.rs
+++ b/tests/mir-opt/const_prop/checked_add.rs
@@ -1,9 +1,12 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
 // compile-flags: -C overflow-checks=on
 
 // EMIT_MIR checked_add.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: assert(!const false,
+    // CHECK: [[x]] = const 2_u32;
     let x: u32 = 1 + 1;
 }
-- 
cgit 1.4.1-3-g733a5


From 043d29b58ae4eee0681b1724ed0c20d9ee3ed9e7 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:21:53 +0000
Subject: FileCheck and rename const_prop_fails_gracefully.

---
 ...ails_gracefully.main.ConstProp.panic-abort.diff | 38 ----------------------
 ...ils_gracefully.main.ConstProp.panic-unwind.diff | 38 ----------------------
 .../const_prop/const_prop_fails_gracefully.rs      | 12 -------
 ..._expose_address.main.ConstProp.panic-abort.diff | 38 ++++++++++++++++++++++
 ...expose_address.main.ConstProp.panic-unwind.diff | 38 ++++++++++++++++++++++
 tests/mir-opt/const_prop/pointer_expose_address.rs | 18 ++++++++++
 6 files changed, 94 insertions(+), 88 deletions(-)
 delete mode 100644 tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-abort.diff
 delete mode 100644 tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-unwind.diff
 delete mode 100644 tests/mir-opt/const_prop/const_prop_fails_gracefully.rs
 create mode 100644 tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-abort.diff
 create mode 100644 tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-unwind.diff
 create mode 100644 tests/mir-opt/const_prop/pointer_expose_address.rs

diff --git a/tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-abort.diff
deleted file mode 100644
index bd1de7476a2..00000000000
--- a/tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-abort.diff
+++ /dev/null
@@ -1,38 +0,0 @@
-- // MIR for `main` before ConstProp
-+ // MIR for `main` after ConstProp
-  
-  fn main() -> () {
-      let mut _0: ();
-      let _1: usize;
-      let mut _2: *const i32;
-      let _3: &i32;
-      let _4: ();
-      let mut _5: usize;
-      scope 1 {
-          debug x => _1;
-      }
-  
-      bb0: {
-          StorageLive(_1);
-          StorageLive(_2);
-          StorageLive(_3);
-          _3 = const _;
-          _2 = &raw const (*_3);
-          _1 = move _2 as usize (PointerExposeAddress);
-          StorageDead(_2);
-          StorageDead(_3);
-          StorageLive(_4);
-          StorageLive(_5);
-          _5 = _1;
-          _4 = read(move _5) -> [return: bb1, unwind unreachable];
-      }
-  
-      bb1: {
-          StorageDead(_5);
-          StorageDead(_4);
-          _0 = const ();
-          StorageDead(_1);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-unwind.diff
deleted file mode 100644
index 850b743feb1..00000000000
--- a/tests/mir-opt/const_prop/const_prop_fails_gracefully.main.ConstProp.panic-unwind.diff
+++ /dev/null
@@ -1,38 +0,0 @@
-- // MIR for `main` before ConstProp
-+ // MIR for `main` after ConstProp
-  
-  fn main() -> () {
-      let mut _0: ();
-      let _1: usize;
-      let mut _2: *const i32;
-      let _3: &i32;
-      let _4: ();
-      let mut _5: usize;
-      scope 1 {
-          debug x => _1;
-      }
-  
-      bb0: {
-          StorageLive(_1);
-          StorageLive(_2);
-          StorageLive(_3);
-          _3 = const _;
-          _2 = &raw const (*_3);
-          _1 = move _2 as usize (PointerExposeAddress);
-          StorageDead(_2);
-          StorageDead(_3);
-          StorageLive(_4);
-          StorageLive(_5);
-          _5 = _1;
-          _4 = read(move _5) -> [return: bb1, unwind continue];
-      }
-  
-      bb1: {
-          StorageDead(_5);
-          StorageDead(_4);
-          _0 = const ();
-          StorageDead(_1);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/const_prop/const_prop_fails_gracefully.rs b/tests/mir-opt/const_prop/const_prop_fails_gracefully.rs
deleted file mode 100644
index 5bd4731bf08..00000000000
--- a/tests/mir-opt/const_prop/const_prop_fails_gracefully.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-// skip-filecheck
-// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
-// unit-test: ConstProp
-#[inline(never)]
-fn read(_: usize) { }
-
-// EMIT_MIR const_prop_fails_gracefully.main.ConstProp.diff
-fn main() {
-    const FOO: &i32 = &1;
-    let x = FOO as *const i32 as usize;
-    read(x);
-}
diff --git a/tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-abort.diff
new file mode 100644
index 00000000000..bd1de7476a2
--- /dev/null
+++ b/tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-abort.diff
@@ -0,0 +1,38 @@
+- // MIR for `main` before ConstProp
++ // MIR for `main` after ConstProp
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: usize;
+      let mut _2: *const i32;
+      let _3: &i32;
+      let _4: ();
+      let mut _5: usize;
+      scope 1 {
+          debug x => _1;
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = const _;
+          _2 = &raw const (*_3);
+          _1 = move _2 as usize (PointerExposeAddress);
+          StorageDead(_2);
+          StorageDead(_3);
+          StorageLive(_4);
+          StorageLive(_5);
+          _5 = _1;
+          _4 = read(move _5) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          _0 = const ();
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-unwind.diff
new file mode 100644
index 00000000000..850b743feb1
--- /dev/null
+++ b/tests/mir-opt/const_prop/pointer_expose_address.main.ConstProp.panic-unwind.diff
@@ -0,0 +1,38 @@
+- // MIR for `main` before ConstProp
++ // MIR for `main` after ConstProp
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: usize;
+      let mut _2: *const i32;
+      let _3: &i32;
+      let _4: ();
+      let mut _5: usize;
+      scope 1 {
+          debug x => _1;
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = const _;
+          _2 = &raw const (*_3);
+          _1 = move _2 as usize (PointerExposeAddress);
+          StorageDead(_2);
+          StorageDead(_3);
+          StorageLive(_4);
+          StorageLive(_5);
+          _5 = _1;
+          _4 = read(move _5) -> [return: bb1, unwind continue];
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          _0 = const ();
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/const_prop/pointer_expose_address.rs b/tests/mir-opt/const_prop/pointer_expose_address.rs
new file mode 100644
index 00000000000..631aac901b9
--- /dev/null
+++ b/tests/mir-opt/const_prop/pointer_expose_address.rs
@@ -0,0 +1,18 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// unit-test: ConstProp
+
+#[inline(never)]
+fn read(_: usize) { }
+
+// EMIT_MIR pointer_expose_address.main.ConstProp.diff
+fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: [[ptr:_.*]] = const _;
+    // CHECK: [[ref:_.*]] = &raw const (*[[ptr]]);
+    // CHECK: [[x:_.*]] = move [[ref]] as usize (PointerExposeAddress);
+    // CHECK: [[arg:_.*]] = [[x]];
+    // CHECK: = read(move [[arg]])
+    const FOO: &i32 = &1;
+    let x = FOO as *const i32 as usize;
+    read(x);
+}
-- 
cgit 1.4.1-3-g733a5


From bf5d114da8a376ee71ea68a6d7f9a5c2e6c4d302 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:28:33 +0000
Subject: FileCheck discriminant.

---
 tests/mir-opt/const_prop/discriminant.rs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/tests/mir-opt/const_prop/discriminant.rs b/tests/mir-opt/const_prop/discriminant.rs
index 11405f38bdc..0ed683d629c 100644
--- a/tests/mir-opt/const_prop/discriminant.rs
+++ b/tests/mir-opt/const_prop/discriminant.rs
@@ -1,6 +1,4 @@
-// skip-filecheck
 // unit-test: ConstProp
-// compile-flags: -O
 
 // FIXME(wesleywiser): Ideally, we could const-prop away all of this and just be left with
 // `let x = 42` but that doesn't work because const-prop doesn't support `Operand::Indirect`
@@ -10,5 +8,18 @@
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
 // EMIT_MIR discriminant.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: bb0: {
+    // CHECK: switchInt(const 1_isize) -> [1: bb1, otherwise: bb3];
+    // CHECK: bb1: {
+    // CHECK: switchInt(const true) -> [0: bb3, otherwise: bb2];
+    // CHECK: bb2: {
+    // CHECK: [[tmp:_.*]] = const 42_i32;
+    // CHECK: goto -> bb4;
+    // CHECK: bb3: {
+    // CHECK: [[tmp]] = const 10_i32;
+    // CHECK: goto -> bb4;
+    // CHECK: bb4: {
+    // CHECK: {{_.*}} = Add(move [[tmp]], const 0_i32);
     let x = (if let Some(true) = Some(true) { 42 } else { 10 }) + 0;
 }
-- 
cgit 1.4.1-3-g733a5


From 6086dd676686c196d26d2a760c653f83fda91e1e Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:29:47 +0000
Subject: FileCheck indirect.

---
 tests/mir-opt/const_prop/indirect.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/indirect.rs b/tests/mir-opt/const_prop/indirect.rs
index 0e6e1d78d1e..d3c42e3eb0b 100644
--- a/tests/mir-opt/const_prop/indirect.rs
+++ b/tests/mir-opt/const_prop/indirect.rs
@@ -1,9 +1,11 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
 // compile-flags: -C overflow-checks=on
 
 // EMIT_MIR indirect.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: [[x]] = const 3_u8;
     let x = (2u32 as u8) + 1;
 }
-- 
cgit 1.4.1-3-g733a5


From 218d8ccf432fef3c5e032e682f5abd844e94df52 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:31:27 +0000
Subject: FileCheck inherit_overflow.

---
 tests/mir-opt/const_prop/inherit_overflow.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/mir-opt/const_prop/inherit_overflow.rs b/tests/mir-opt/const_prop/inherit_overflow.rs
index 41989462deb..5b561ae14ad 100644
--- a/tests/mir-opt/const_prop/inherit_overflow.rs
+++ b/tests/mir-opt/const_prop/inherit_overflow.rs
@@ -1,11 +1,14 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
 // compile-flags: -Zmir-enable-passes=+Inline
 
+// After inlining, this will contain a `CheckedBinaryOp`.
+// Propagating the overflow is ok as codegen will just skip emitting the panic.
 // EMIT_MIR inherit_overflow.main.ConstProp.diff
 fn main() {
-    // After inlining, this will contain a `CheckedBinaryOp`.
-    // Propagating the overflow is ok as codegen will just skip emitting the panic.
+    // CHECK-LABEL: fn main(
+    // CHECK: {{_.*}} = const (0_u8, true);
+    // CHECK: assert(!const true,
+    // CHECK: {{_.*}} = const 0_u8;
     let _ = <u8 as std::ops::Add>::add(255, 1);
 }
-- 
cgit 1.4.1-3-g733a5


From ce9b1e23a58ed1ba3d1f1241d0c323996b5eb430 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:36:48 +0000
Subject: FileCheck issue_66971.

---
 .../const_prop/issue_66971.main.ConstProp.panic-abort.diff    | 11 +++++++++--
 .../const_prop/issue_66971.main.ConstProp.panic-unwind.diff   | 11 +++++++++--
 tests/mir-opt/const_prop/issue_66971.rs                       |  4 ++--
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-abort.diff
index 6484b4b67af..ff93c85e586 100644
--- a/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-abort.diff
+++ b/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-abort.diff
@@ -5,17 +5,24 @@
       let mut _0: ();
       let _1: ();
       let mut _2: ((), u8, u8);
+      let mut _3: ();
   
       bb0: {
+          StorageLive(_1);
           StorageLive(_2);
--         _2 = (const (), const 0_u8, const 0_u8);
--         _1 = encode(move _2) -> [return: bb1, unwind unreachable];
+          StorageLive(_3);
+          _3 = ();
+-         _2 = (move _3, const 0_u8, const 0_u8);
 +         _2 = const ((), 0_u8, 0_u8);
+          StorageDead(_3);
+-         _1 = encode(move _2) -> [return: bb1, unwind unreachable];
 +         _1 = encode(const ((), 0_u8, 0_u8)) -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
           StorageDead(_2);
+          StorageDead(_1);
+          _0 = const ();
           return;
       }
 + }
diff --git a/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-unwind.diff
index b02f0407839..8790aad4559 100644
--- a/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/issue_66971.main.ConstProp.panic-unwind.diff
@@ -5,17 +5,24 @@
       let mut _0: ();
       let _1: ();
       let mut _2: ((), u8, u8);
+      let mut _3: ();
   
       bb0: {
+          StorageLive(_1);
           StorageLive(_2);
--         _2 = (const (), const 0_u8, const 0_u8);
--         _1 = encode(move _2) -> [return: bb1, unwind continue];
+          StorageLive(_3);
+          _3 = ();
+-         _2 = (move _3, const 0_u8, const 0_u8);
 +         _2 = const ((), 0_u8, 0_u8);
+          StorageDead(_3);
+-         _1 = encode(move _2) -> [return: bb1, unwind continue];
 +         _1 = encode(const ((), 0_u8, 0_u8)) -> [return: bb1, unwind continue];
       }
   
       bb1: {
           StorageDead(_2);
+          StorageDead(_1);
+          _0 = const ();
           return;
       }
 + }
diff --git a/tests/mir-opt/const_prop/issue_66971.rs b/tests/mir-opt/const_prop/issue_66971.rs
index 386c95b5b69..49d598ff230 100644
--- a/tests/mir-opt/const_prop/issue_66971.rs
+++ b/tests/mir-opt/const_prop/issue_66971.rs
@@ -1,7 +1,5 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
-// compile-flags: -Z mir-opt-level=3
 
 // Due to a bug in propagating scalar pairs the assertion below used to fail. In the expected
 // outputs below, after ConstProp this is how _2 would look like with the bug:
@@ -16,5 +14,7 @@ fn encode(this: ((), u8, u8)) {
 
 // EMIT_MIR issue_66971.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: = encode(const ((), 0_u8, 0_u8))
     encode(((), 0, 0));
 }
-- 
cgit 1.4.1-3-g733a5


From 8e9b912c4cbce7cf1eae011a2dcfefcb916c416c Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:38:07 +0000
Subject: FileCheck issue_67019.

---
 tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-abort.diff  | 3 +++
 tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-unwind.diff | 3 +++
 tests/mir-opt/const_prop/issue_67019.rs                               | 4 ++--
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-abort.diff
index c1ef453e9df..3de9cdd79bc 100644
--- a/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-abort.diff
+++ b/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-abort.diff
@@ -8,6 +8,7 @@
       let mut _3: (u8, u8);
   
       bb0: {
+          StorageLive(_1);
           StorageLive(_2);
           StorageLive(_3);
 -         _3 = (const 1_u8, const 2_u8);
@@ -21,6 +22,8 @@
   
       bb1: {
           StorageDead(_2);
+          StorageDead(_1);
+          _0 = const ();
           return;
       }
 + }
diff --git a/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-unwind.diff
index 53cdcc18167..72cf48b5cba 100644
--- a/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/issue_67019.main.ConstProp.panic-unwind.diff
@@ -8,6 +8,7 @@
       let mut _3: (u8, u8);
   
       bb0: {
+          StorageLive(_1);
           StorageLive(_2);
           StorageLive(_3);
 -         _3 = (const 1_u8, const 2_u8);
@@ -21,6 +22,8 @@
   
       bb1: {
           StorageDead(_2);
+          StorageDead(_1);
+          _0 = const ();
           return;
       }
 + }
diff --git a/tests/mir-opt/const_prop/issue_67019.rs b/tests/mir-opt/const_prop/issue_67019.rs
index 2f61298bb98..f0a09e6e852 100644
--- a/tests/mir-opt/const_prop/issue_67019.rs
+++ b/tests/mir-opt/const_prop/issue_67019.rs
@@ -1,7 +1,5 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
-// compile-flags: -Z mir-opt-level=3
 
 // This used to ICE in const-prop
 
@@ -11,5 +9,7 @@ fn test(this: ((u8, u8),)) {
 
 // EMIT_MIR issue_67019.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: = test(const ((1_u8, 2_u8),))
     test(((1, 2),));
 }
-- 
cgit 1.4.1-3-g733a5


From 902a3e2e7540019337dcbba0f4aa160a6b0495b3 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:39:26 +0000
Subject: FileCheck mult_by_zero.

---
 tests/mir-opt/const_prop/mult_by_zero.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/mir-opt/const_prop/mult_by_zero.rs b/tests/mir-opt/const_prop/mult_by_zero.rs
index 47e15205ea9..2e9c63a1ca1 100644
--- a/tests/mir-opt/const_prop/mult_by_zero.rs
+++ b/tests/mir-opt/const_prop/mult_by_zero.rs
@@ -1,9 +1,10 @@
-// skip-filecheck
 // unit-test: ConstProp
 
 // EMIT_MIR mult_by_zero.test.ConstProp.diff
-fn test(x : i32) -> i32 {
-  x * 0
+fn test(x: i32) -> i32 {
+    // CHECK: fn test(
+    // CHECK: _0 = const 0_i32;
+    x * 0
 }
 
 fn main() {
-- 
cgit 1.4.1-3-g733a5


From ea9f9683335f0edf4cbafe5b1ebe663c1864ea54 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:49:28 +0000
Subject: FileCheck mutable_variable.

---
 tests/mir-opt/const_prop/mutable_variable.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/mutable_variable.rs b/tests/mir-opt/const_prop/mutable_variable.rs
index 175d63d46f5..6c74ea5b9f4 100644
--- a/tests/mir-opt/const_prop/mutable_variable.rs
+++ b/tests/mir-opt/const_prop/mutable_variable.rs
@@ -1,8 +1,13 @@
-// skip-filecheck
 // unit-test: ConstProp
 
 // EMIT_MIR mutable_variable.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: [[x]] = const 42_i32;
+    // CHECK: [[x]] = const 99_i32;
+    // CHECK: [[y]] = const 99_i32;
     let mut x = 42;
     x = 99;
     let y = x;
-- 
cgit 1.4.1-3-g733a5


From 03c5ad1549385ae8c8e3f6b9530c473ac34e0d31 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:49:38 +0000
Subject: FileCheck mutable_variable_aggregate.

---
 tests/mir-opt/const_prop/mutable_variable_aggregate.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/mutable_variable_aggregate.rs b/tests/mir-opt/const_prop/mutable_variable_aggregate.rs
index f926771ae38..a3829650290 100644
--- a/tests/mir-opt/const_prop/mutable_variable_aggregate.rs
+++ b/tests/mir-opt/const_prop/mutable_variable_aggregate.rs
@@ -1,8 +1,13 @@
-// skip-filecheck
 // unit-test: ConstProp
 
 // EMIT_MIR mutable_variable_aggregate.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: [[x]] = const (42_i32, 43_i32);
+    // CHECK: ([[x]].1: i32) = const 99_i32;
+    // CHECK: [[y]] = const (42_i32, 99_i32);
     let mut x = (42, 43);
     x.1 = 99;
     let y = x;
-- 
cgit 1.4.1-3-g733a5


From 3e169abc1b2331c608b9c085417d2ecd64293235 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:49:50 +0000
Subject: FileCheck mutable_variable_aggregate_mut_ref.

---
 tests/mir-opt/const_prop/mutable_variable_aggregate_mut_ref.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/mutable_variable_aggregate_mut_ref.rs b/tests/mir-opt/const_prop/mutable_variable_aggregate_mut_ref.rs
index a81aa7b4979..60f414ae286 100644
--- a/tests/mir-opt/const_prop/mutable_variable_aggregate_mut_ref.rs
+++ b/tests/mir-opt/const_prop/mutable_variable_aggregate_mut_ref.rs
@@ -1,8 +1,15 @@
-// skip-filecheck
 // unit-test: ConstProp
 
 // EMIT_MIR mutable_variable_aggregate_mut_ref.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug z => [[z:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: [[x]] = (const 42_i32, const 43_i32);
+    // CHECK: [[z]] = &mut [[x]];
+    // CHECK: ((*[[z]]).1: i32) = const 99_i32;
+    // CHECK: [[y]] = [[x]];
     let mut x = (42, 43);
     let z = &mut x;
     z.1 = 99;
-- 
cgit 1.4.1-3-g733a5


From d91bb5074e4f926bb9c2b2b43237bcf8d3c512b6 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:50:00 +0000
Subject: FileCheck mutable_variable_no_prop.

---
 tests/mir-opt/const_prop/mutable_variable_no_prop.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/mutable_variable_no_prop.rs b/tests/mir-opt/const_prop/mutable_variable_no_prop.rs
index a7aeeccd861..49e9a701581 100644
--- a/tests/mir-opt/const_prop/mutable_variable_no_prop.rs
+++ b/tests/mir-opt/const_prop/mutable_variable_no_prop.rs
@@ -1,10 +1,17 @@
-// skip-filecheck
 // unit-test: ConstProp
 
+// Verify that we do not propagate the contents of this mutable static.
 static mut STATIC: u32 = 0x42424242;
 
 // EMIT_MIR mutable_variable_no_prop.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: [[x]] = const 42_u32;
+    // CHECK: [[tmp:_.*]] = (*{{_.*}});
+    // CHECK: [[x]] = move [[tmp]];
+    // CHECK: [[y]] = [[x]];
     let mut x = 42;
     unsafe {
         x = STATIC;
-- 
cgit 1.4.1-3-g733a5


From 6a8eea8f5b7feb9e1e87cf1673c11eab93307001 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:51:23 +0000
Subject: FileCheck mutable_variable_aggregate_partial_read.

---
 .../mir-opt/const_prop/mutable_variable_aggregate_partial_read.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/mutable_variable_aggregate_partial_read.rs b/tests/mir-opt/const_prop/mutable_variable_aggregate_partial_read.rs
index 54a5d922321..888fcde2de6 100644
--- a/tests/mir-opt/const_prop/mutable_variable_aggregate_partial_read.rs
+++ b/tests/mir-opt/const_prop/mutable_variable_aggregate_partial_read.rs
@@ -1,9 +1,15 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
 
 // EMIT_MIR mutable_variable_aggregate_partial_read.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: [[x]] = foo()
+    // CHECK: ([[x]].1: i32) = const 99_i32;
+    // CHECK: ([[x]].0: i32) = const 42_i32;
+    // CHECK: [[y]] = const 99_i32;
     let mut x: (i32, i32) = foo();
     x.1 = 99;
     x.0 = 42;
-- 
cgit 1.4.1-3-g733a5


From 45dd5d6bf340b48cf2a00755da4ca9dd7b6eb640 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:54:56 +0000
Subject: FileCheck mutable_variable_unprop_assign.

---
 tests/mir-opt/const_prop/mutable_variable_unprop_assign.rs | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tests/mir-opt/const_prop/mutable_variable_unprop_assign.rs b/tests/mir-opt/const_prop/mutable_variable_unprop_assign.rs
index 6bdb136a949..04e347fc03d 100644
--- a/tests/mir-opt/const_prop/mutable_variable_unprop_assign.rs
+++ b/tests/mir-opt/const_prop/mutable_variable_unprop_assign.rs
@@ -1,14 +1,24 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
 
 // EMIT_MIR mutable_variable_unprop_assign.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: debug z => [[z:_.*]];
+    // CHECK: [[a]] = foo()
+    // CHECK: [[x]] = const (1_i32, 2_i32);
+    // CHECK: [[tmp:_.*]] = [[a]];
+    // CHECK: ([[x]].1: i32) = move [[tmp]];
+    // CHECK: [[y]] = ([[x]].1: i32);
+    // CHECK: [[z]] = const 1_i32;
     let a = foo();
     let mut x: (i32, i32) = (1, 2);
     x.1 = a;
     let y = x.1;
-    let z = x.0; // this could theoretically be allowed, but we can't handle it right now
+    let z = x.0;
 }
 
 #[inline(never)]
-- 
cgit 1.4.1-3-g733a5


From c8c9207e4c29a42f69408e92ca95fc34f7bf04c6 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:56:11 +0000
Subject: FileCheck read_immutable_static.

---
 tests/mir-opt/const_prop/read_immutable_static.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/read_immutable_static.rs b/tests/mir-opt/const_prop/read_immutable_static.rs
index a8d8cfacc7c..0fa18dd101a 100644
--- a/tests/mir-opt/const_prop/read_immutable_static.rs
+++ b/tests/mir-opt/const_prop/read_immutable_static.rs
@@ -1,9 +1,11 @@
-// skip-filecheck
 // unit-test: ConstProp
 
 static FOO: u8 = 2;
 
 // EMIT_MIR read_immutable_static.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: [[x]] = const 4_u8;
     let x = FOO + FOO;
 }
-- 
cgit 1.4.1-3-g733a5


From 6baec3ccc27a719e12e5e5ad67995c0cd86f60e6 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 20:58:35 +0000
Subject: FileCheck ref_deref.

---
 tests/mir-opt/const_prop/ref_deref.main.ConstProp.diff         | 5 ++++-
 tests/mir-opt/const_prop/ref_deref.rs                          | 8 +++++---
 tests/mir-opt/const_prop/ref_deref_project.main.ConstProp.diff | 5 ++++-
 tests/mir-opt/const_prop/ref_deref_project.rs                  | 9 ++++++---
 4 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/tests/mir-opt/const_prop/ref_deref.main.ConstProp.diff b/tests/mir-opt/const_prop/ref_deref.main.ConstProp.diff
index 6b897a88181..a54ae8d2fdd 100644
--- a/tests/mir-opt/const_prop/ref_deref.main.ConstProp.diff
+++ b/tests/mir-opt/const_prop/ref_deref.main.ConstProp.diff
@@ -7,6 +7,9 @@
       let mut _2: &i32;
       let _3: i32;
       let mut _4: &i32;
+      scope 1 {
+          debug a => _1;
+      }
   
       bb0: {
           StorageLive(_1);
@@ -15,8 +18,8 @@
           _2 = &(*_4);
           _1 = (*_2);
           StorageDead(_2);
-          StorageDead(_1);
           _0 = const ();
+          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/ref_deref.rs b/tests/mir-opt/const_prop/ref_deref.rs
index f2fa024f722..5bceae749ff 100644
--- a/tests/mir-opt/const_prop/ref_deref.rs
+++ b/tests/mir-opt/const_prop/ref_deref.rs
@@ -1,7 +1,9 @@
-// skip-filecheck
 // unit-test: ConstProp
-// EMIT_MIR ref_deref.main.ConstProp.diff
 
+// EMIT_MIR ref_deref.main.ConstProp.diff
 fn main() {
-    *(&4);
+    // CHECK-LABEL: fn main(
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: [[a]] = (*{{_.*}});
+    let a = *(&4);
 }
diff --git a/tests/mir-opt/const_prop/ref_deref_project.main.ConstProp.diff b/tests/mir-opt/const_prop/ref_deref_project.main.ConstProp.diff
index 7f616166573..05a4e17742d 100644
--- a/tests/mir-opt/const_prop/ref_deref_project.main.ConstProp.diff
+++ b/tests/mir-opt/const_prop/ref_deref_project.main.ConstProp.diff
@@ -7,6 +7,9 @@
       let mut _2: &i32;
       let _3: (i32, i32);
       let mut _4: &(i32, i32);
+      scope 1 {
+          debug a => _1;
+      }
   
       bb0: {
           StorageLive(_1);
@@ -15,8 +18,8 @@
           _2 = &((*_4).1: i32);
           _1 = (*_2);
           StorageDead(_2);
-          StorageDead(_1);
           _0 = const ();
+          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/ref_deref_project.rs b/tests/mir-opt/const_prop/ref_deref_project.rs
index 1b9e0acb2c0..4b5c6730316 100644
--- a/tests/mir-opt/const_prop/ref_deref_project.rs
+++ b/tests/mir-opt/const_prop/ref_deref_project.rs
@@ -1,7 +1,10 @@
-// skip-filecheck
+// This does not currently propagate (#67862)
 // unit-test: ConstProp
-// EMIT_MIR ref_deref_project.main.ConstProp.diff
 
+// EMIT_MIR ref_deref_project.main.ConstProp.diff
 fn main() {
-    *(&(4, 5).1); // This does not currently propagate (#67862)
+    // CHECK-LABEL: fn main(
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: [[a]] = (*{{_.*}});
+    let a = *(&(4, 5).1);
 }
-- 
cgit 1.4.1-3-g733a5


From 343ef6a9cb8425e394643b735a9a4ef361067fe2 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:00:59 +0000
Subject: FileCheck reify_fn_ptr.

---
 tests/mir-opt/const_prop/reify_fn_ptr.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/reify_fn_ptr.rs b/tests/mir-opt/const_prop/reify_fn_ptr.rs
index da7de80c5f4..33fdd4142c1 100644
--- a/tests/mir-opt/const_prop/reify_fn_ptr.rs
+++ b/tests/mir-opt/const_prop/reify_fn_ptr.rs
@@ -1,7 +1,10 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR reify_fn_ptr.main.ConstProp.diff
 
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: [[ptr:_.*]] = main as fn() (PointerCoercion(ReifyFnPointer));
+    // CHECK: [[addr:_.*]] = move [[ptr]] as usize (PointerExposeAddress);
+    // CHECK: [[back:_.*]] = move [[addr]] as *const fn() (PointerFromExposedAddress);
     let _ = main as usize as *const fn();
 }
-- 
cgit 1.4.1-3-g733a5


From f3743aec51c3817ad93854a06173de1db3ab8660 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:01:48 +0000
Subject: FileCheck repeat.

---
 tests/mir-opt/const_prop/repeat.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/repeat.rs b/tests/mir-opt/const_prop/repeat.rs
index 92194d6bb58..9f688bbb53e 100644
--- a/tests/mir-opt/const_prop/repeat.rs
+++ b/tests/mir-opt/const_prop/repeat.rs
@@ -1,9 +1,11 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
 
 // EMIT_MIR repeat.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: [[x]] = const 42_u32;
     let x: u32 = [42; 8][2] + 0;
 }
-- 
cgit 1.4.1-3-g733a5


From 3e90c1b434e8c8a02d42ac01c8d79d0c602af418 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:02:42 +0000
Subject: FileCheck scalar_literal_propagation.

---
 tests/mir-opt/const_prop/scalar_literal_propagation.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/scalar_literal_propagation.rs b/tests/mir-opt/const_prop/scalar_literal_propagation.rs
index 9dcddf7c770..782cd35d422 100644
--- a/tests/mir-opt/const_prop/scalar_literal_propagation.rs
+++ b/tests/mir-opt/const_prop/scalar_literal_propagation.rs
@@ -1,8 +1,10 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
 // EMIT_MIR scalar_literal_propagation.main.ConstProp.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: = consume(const 1_u32)
     let x = 1;
     consume(x);
 }
-- 
cgit 1.4.1-3-g733a5


From 19767eb7a6fba1364d32d56669696b4373ac7882 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:04:15 +0000
Subject: FileCheck slice_len.

---
 .../const_prop/slice_len.main.ConstProp.32bit.panic-abort.diff   | 5 ++++-
 .../const_prop/slice_len.main.ConstProp.32bit.panic-unwind.diff  | 5 ++++-
 .../const_prop/slice_len.main.ConstProp.64bit.panic-abort.diff   | 5 ++++-
 .../const_prop/slice_len.main.ConstProp.64bit.panic-unwind.diff  | 5 ++++-
 tests/mir-opt/const_prop/slice_len.rs                            | 9 ++++++---
 5 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-abort.diff b/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-abort.diff
index c2e1288b41e..7d5d036f460 100644
--- a/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-abort.diff
+++ b/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-abort.diff
@@ -12,6 +12,9 @@
       let mut _7: usize;
       let mut _8: bool;
       let mut _9: &[u32; 3];
+      scope 1 {
+          debug a => _1;
+      }
   
       bb0: {
           StorageLive(_1);
@@ -39,8 +42,8 @@
           StorageDead(_6);
           StorageDead(_4);
           StorageDead(_2);
-          StorageDead(_1);
           _0 = const ();
+          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-unwind.diff b/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-unwind.diff
index 23646c3c976..fa4c5a71be5 100644
--- a/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/slice_len.main.ConstProp.32bit.panic-unwind.diff
@@ -12,6 +12,9 @@
       let mut _7: usize;
       let mut _8: bool;
       let mut _9: &[u32; 3];
+      scope 1 {
+          debug a => _1;
+      }
   
       bb0: {
           StorageLive(_1);
@@ -39,8 +42,8 @@
           StorageDead(_6);
           StorageDead(_4);
           StorageDead(_2);
-          StorageDead(_1);
           _0 = const ();
+          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-abort.diff b/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-abort.diff
index c2e1288b41e..7d5d036f460 100644
--- a/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-abort.diff
+++ b/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-abort.diff
@@ -12,6 +12,9 @@
       let mut _7: usize;
       let mut _8: bool;
       let mut _9: &[u32; 3];
+      scope 1 {
+          debug a => _1;
+      }
   
       bb0: {
           StorageLive(_1);
@@ -39,8 +42,8 @@
           StorageDead(_6);
           StorageDead(_4);
           StorageDead(_2);
-          StorageDead(_1);
           _0 = const ();
+          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-unwind.diff b/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-unwind.diff
index 23646c3c976..fa4c5a71be5 100644
--- a/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/slice_len.main.ConstProp.64bit.panic-unwind.diff
@@ -12,6 +12,9 @@
       let mut _7: usize;
       let mut _8: bool;
       let mut _9: &[u32; 3];
+      scope 1 {
+          debug a => _1;
+      }
   
       bb0: {
           StorageLive(_1);
@@ -39,8 +42,8 @@
           StorageDead(_6);
           StorageDead(_4);
           StorageDead(_2);
-          StorageDead(_1);
           _0 = const ();
+          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/slice_len.rs b/tests/mir-opt/const_prop/slice_len.rs
index 3b551b6b173..0bf44272698 100644
--- a/tests/mir-opt/const_prop/slice_len.rs
+++ b/tests/mir-opt/const_prop/slice_len.rs
@@ -1,10 +1,13 @@
-// skip-filecheck
-// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: ConstProp
 // compile-flags: -Zmir-enable-passes=+InstSimplify
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
 
 // EMIT_MIR slice_len.main.ConstProp.diff
 fn main() {
-    (&[1u32, 2, 3] as &[u32])[1];
+    // CHECK-LABEL: fn main(
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: assert(const true,
+    // CHECK: [[a]] = const 2_u32;
+    let a = (&[1u32, 2, 3] as &[u32])[1];
 }
-- 
cgit 1.4.1-3-g733a5


From a12027e1283151f6399c1ad1f7822f46ea5c1a24 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:06:48 +0000
Subject: FileCheck switch_int.

---
 tests/mir-opt/const_prop/switch_int.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/switch_int.rs b/tests/mir-opt/const_prop/switch_int.rs
index 7ec56e11e85..d1cbaae49aa 100644
--- a/tests/mir-opt/const_prop/switch_int.rs
+++ b/tests/mir-opt/const_prop/switch_int.rs
@@ -1,13 +1,20 @@
-// skip-filecheck
 // unit-test: ConstProp
 // compile-flags: -Zmir-enable-passes=+SimplifyConstCondition-after-const-prop
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
 #[inline(never)]
 fn foo(_: i32) { }
 
 // EMIT_MIR switch_int.main.ConstProp.diff
 // EMIT_MIR switch_int.main.SimplifyConstCondition-after-const-prop.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: bb0: {
+    // CHECK-NOT: switchInt(
+    // CHECK: goto -> [[bb:bb.*]];
+    // CHECK: [[bb]]: {
+    // CHECK-NOT: _0 = foo(const -1_i32)
+    // CHECK: _0 = foo(const 0_i32)
     match 1 {
         1 => foo(0),
         _ => foo(-1),
-- 
cgit 1.4.1-3-g733a5


From 87522d0007562be5d68e4838b2f6f7b579f8c2af Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:07:30 +0000
Subject: FileCheck return_place.

---
 tests/mir-opt/const_prop/return_place.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/return_place.rs b/tests/mir-opt/const_prop/return_place.rs
index 1263de7931f..286543abb99 100644
--- a/tests/mir-opt/const_prop/return_place.rs
+++ b/tests/mir-opt/const_prop/return_place.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // compile-flags: -C overflow-checks=on
@@ -6,6 +5,8 @@
 // EMIT_MIR return_place.add.ConstProp.diff
 // EMIT_MIR return_place.add.PreCodegen.before.mir
 fn add() -> u32 {
+    // CHECK-LABEL: fn add(
+    // CHECK: _0 = const 4_u32;
     2 + 2
 }
 
-- 
cgit 1.4.1-3-g733a5


From c00068e49fd215f686d66a8d353c4336dc5bbf76 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:08:55 +0000
Subject: FileCheck tuple_literal_propagation.

---
 tests/mir-opt/const_prop/tuple_literal_propagation.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/mir-opt/const_prop/tuple_literal_propagation.rs b/tests/mir-opt/const_prop/tuple_literal_propagation.rs
index e0bc6e1be37..dfc4a6f3fbb 100644
--- a/tests/mir-opt/const_prop/tuple_literal_propagation.rs
+++ b/tests/mir-opt/const_prop/tuple_literal_propagation.rs
@@ -1,10 +1,11 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // EMIT_MIR tuple_literal_propagation.main.ConstProp.diff
+
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: = consume(const (1_u32, 2_u32))
     let x = (1, 2);
-
     consume(x);
 }
 
-- 
cgit 1.4.1-3-g733a5


From 30a95b7c0a97f0aa45c3b090548ba7977ba4791a Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 2 Dec 2023 21:10:04 +0000
Subject: FileCheck while_let_loops.

---
 tests/mir-opt/const_prop/while_let_loops.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/const_prop/while_let_loops.rs b/tests/mir-opt/const_prop/while_let_loops.rs
index 39081c3550a..8b2a73438d6 100644
--- a/tests/mir-opt/const_prop/while_let_loops.rs
+++ b/tests/mir-opt/const_prop/while_let_loops.rs
@@ -1,8 +1,9 @@
-// skip-filecheck
 // unit-test: ConstProp
 // EMIT_MIR while_let_loops.change_loop_body.ConstProp.diff
 
 pub fn change_loop_body() {
+    // CHECK-LABEL: fn change_loop_body(
+    // CHECK: switchInt(const 0_isize)
     let mut _x = 0;
     while let Some(0u32) = None {
         _x = 1;
-- 
cgit 1.4.1-3-g733a5


From 80bdcbf50a63845dd3cfeb05751ba3dcbd1025b8 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 27 Nov 2023 03:15:56 +0100
Subject: Parse a pattern with no arm

---
 compiler/rustc_ast/src/ast.rs                      |  22 +-
 compiler/rustc_ast/src/mut_visit.rs                |   2 +-
 compiler/rustc_ast/src/visit.rs                    |   2 +-
 compiler/rustc_ast_lowering/src/expr.rs            |  29 ++-
 compiler/rustc_ast_pretty/src/pprust/state/expr.rs |  37 ++--
 .../src/deriving/cmp/partial_ord.rs                |   2 +-
 compiler/rustc_expand/src/build.rs                 |   2 +-
 compiler/rustc_expand/src/placeholders.rs          |   2 +-
 compiler/rustc_lint/src/builtin.rs                 |   6 +-
 compiler/rustc_lint/src/unused.rs                  |  20 +-
 compiler/rustc_parse/src/parser/expr.rs            | 242 ++++++++++++---------
 compiler/rustc_parse/src/parser/pat.rs             |   4 +-
 compiler/rustc_resolve/src/late.rs                 |   2 +-
 .../clippy_lints/src/non_expressive_names.rs       |   4 +-
 .../clippy/clippy_lints/src/redundant_else.rs      |   4 +-
 src/tools/clippy/clippy_utils/src/ast_utils.rs     |   2 +-
 src/tools/rustfmt/src/matches.rs                   |   8 +-
 src/tools/rustfmt/src/spanned.rs                   |   7 +-
 .../range_pat_interactions1.rs                     |   2 +-
 .../range_pat_interactions1.stderr                 |   4 +-
 .../range_pat_interactions2.rs                     |   2 +-
 .../range_pat_interactions2.stderr                 |   4 +-
 tests/ui/never_patterns/check.rs                   |   3 +-
 tests/ui/never_patterns/check.stderr               |  41 +++-
 tests/ui/never_patterns/parse.rs                   |  68 ++++++
 tests/ui/never_patterns/parse.stderr               |  26 +++
 .../ui/parser/attribute/attr-stmt-expr-attr-bad.rs |   6 +-
 .../attribute/attr-stmt-expr-attr-bad.stderr       |  12 +-
 tests/ui/parser/issues/issue-24375.rs              |   2 +-
 tests/ui/parser/issues/issue-24375.stderr          |   4 +-
 tests/ui/parser/macro/macro-expand-to-match-arm.rs |   6 +-
 .../parser/macro/macro-expand-to-match-arm.stderr  |  11 +-
 tests/ui/parser/match-arm-without-body.rs          |  11 +-
 tests/ui/parser/match-arm-without-body.stderr      |  84 +------
 tests/ui/parser/pat-lt-bracket-1.rs                |   2 +-
 tests/ui/parser/pat-lt-bracket-1.stderr            |   4 +-
 36 files changed, 415 insertions(+), 274 deletions(-)
 create mode 100644 tests/ui/never_patterns/parse.rs
 create mode 100644 tests/ui/never_patterns/parse.stderr

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index cba9121aa5a..62bacf97f49 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -658,6 +658,24 @@ impl Pat {
     pub fn is_rest(&self) -> bool {
         matches!(self.kind, PatKind::Rest)
     }
+
+    /// Could this be a never pattern? I.e. is it a never pattern modulo macro invocations that
+    /// might return never patterns?
+    pub fn could_be_never_pattern(&self) -> bool {
+        let mut could_be_never_pattern = false;
+        self.walk(&mut |pat| match &pat.kind {
+            PatKind::Never | PatKind::MacCall(_) => {
+                could_be_never_pattern = true;
+                false
+            }
+            PatKind::Or(s) => {
+                could_be_never_pattern = s.iter().all(|p| p.could_be_never_pattern());
+                false
+            }
+            _ => true,
+        });
+        could_be_never_pattern
+    }
 }
 
 /// A single field in a struct pattern.
@@ -1080,8 +1098,8 @@ pub struct Arm {
     pub pat: P<Pat>,
     /// Match arm guard, e.g. `n > 10` in `match foo { n if n > 10 => {}, _ => {} }`
     pub guard: Option<P<Expr>>,
-    /// Match arm body.
-    pub body: P<Expr>,
+    /// Match arm body. Omitted if the pattern is a never pattern.
+    pub body: Option<P<Expr>>,
     pub span: Span,
     pub id: NodeId,
     pub is_placeholder: bool,
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index 8ce86bf9ecf..e909e8dc774 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -453,7 +453,7 @@ pub fn noop_flat_map_arm<T: MutVisitor>(mut arm: Arm, vis: &mut T) -> SmallVec<[
     vis.visit_id(id);
     vis.visit_pat(pat);
     visit_opt(guard, |guard| vis.visit_expr(guard));
-    vis.visit_expr(body);
+    visit_opt(body, |body| vis.visit_expr(body));
     vis.visit_span(span);
     smallvec![arm]
 }
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index 9dbadcb49d3..773a7ebc707 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -951,7 +951,7 @@ pub fn walk_param<'a, V: Visitor<'a>>(visitor: &mut V, param: &'a Param) {
 pub fn walk_arm<'a, V: Visitor<'a>>(visitor: &mut V, arm: &'a Arm) {
     visitor.visit_pat(&arm.pat);
     walk_list!(visitor, visit_expr, &arm.guard);
-    visitor.visit_expr(&arm.body);
+    walk_list!(visitor, visit_expr, &arm.body);
     walk_list!(visitor, visit_attribute, &arm.attrs);
 }
 
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index be5671f1bf7..b688950f5a3 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -566,13 +566,28 @@ impl<'hir> LoweringContext<'_, 'hir> {
         });
         let hir_id = self.next_id();
         self.lower_attrs(hir_id, &arm.attrs);
-        hir::Arm {
-            hir_id,
-            pat,
-            guard,
-            body: self.lower_expr(&arm.body),
-            span: self.lower_span(arm.span),
-        }
+        let body = if let Some(body) = &arm.body {
+            self.lower_expr(body)
+        } else {
+            // An arm without a body, meant for never patterns.
+            // We add a fake `loop {}` arm body so that it typecks to `!`.
+            // FIXME(never_patterns): Desugar into a call to `unreachable_unchecked`.
+            let span = pat.span;
+            let block = self.arena.alloc(hir::Block {
+                stmts: &[],
+                expr: None,
+                hir_id: self.next_id(),
+                rules: hir::BlockCheckMode::DefaultBlock,
+                span,
+                targeted_by_break: false,
+            });
+            self.arena.alloc(hir::Expr {
+                hir_id: self.next_id(),
+                kind: hir::ExprKind::Loop(block, None, hir::LoopSource::Loop, span),
+                span,
+            })
+        };
+        hir::Arm { hir_id, pat, guard, body, span: self.lower_span(arm.span) }
     }
 
     /// Lower an `async` construct to a coroutine that implements `Future`.
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index 45a55e20ca7..d7dbc7ac035 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -631,28 +631,33 @@ impl<'a> State<'a> {
             self.print_expr(e);
             self.space();
         }
-        self.word_space("=>");
 
-        match &arm.body.kind {
-            ast::ExprKind::Block(blk, opt_label) => {
-                if let Some(label) = opt_label {
-                    self.print_ident(label.ident);
-                    self.word_space(":");
-                }
+        if let Some(body) = &arm.body {
+            self.word_space("=>");
+
+            match &body.kind {
+                ast::ExprKind::Block(blk, opt_label) => {
+                    if let Some(label) = opt_label {
+                        self.print_ident(label.ident);
+                        self.word_space(":");
+                    }
 
-                // The block will close the pattern's ibox.
-                self.print_block_unclosed_indent(blk);
+                    // The block will close the pattern's ibox.
+                    self.print_block_unclosed_indent(blk);
 
-                // If it is a user-provided unsafe block, print a comma after it.
-                if let BlockCheckMode::Unsafe(ast::UserProvided) = blk.rules {
+                    // If it is a user-provided unsafe block, print a comma after it.
+                    if let BlockCheckMode::Unsafe(ast::UserProvided) = blk.rules {
+                        self.word(",");
+                    }
+                }
+                _ => {
+                    self.end(); // Close the ibox for the pattern.
+                    self.print_expr(body);
                     self.word(",");
                 }
             }
-            _ => {
-                self.end(); // Close the ibox for the pattern.
-                self.print_expr(&arm.body);
-                self.word(",");
-            }
+        } else {
+            self.word(",");
         }
         self.end(); // Close enclosing cbox.
     }
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
index f3164bd2c2a..7f5589210d4 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
@@ -136,7 +136,7 @@ fn cs_partial_cmp(
                     && let Some(last) = arms.last_mut()
                     && let PatKind::Wild = last.pat.kind
                 {
-                    last.body = expr2;
+                    last.body = Some(expr2);
                     expr1
                 } else {
                     let eq_arm = cx.arm(
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index 9a8d0d691f0..0996a45d900 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -505,7 +505,7 @@ impl<'a> ExtCtxt<'a> {
             attrs: AttrVec::new(),
             pat,
             guard: None,
-            body: expr,
+            body: Some(expr),
             span,
             id: ast::DUMMY_NODE_ID,
             is_placeholder: false,
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
index 1292a855230..ded0baa9563 100644
--- a/compiler/rustc_expand/src/placeholders.rs
+++ b/compiler/rustc_expand/src/placeholders.rs
@@ -119,7 +119,7 @@ pub fn placeholder(
         }]),
         AstFragmentKind::Arms => AstFragment::Arms(smallvec![ast::Arm {
             attrs: Default::default(),
-            body: expr_placeholder(),
+            body: Some(expr_placeholder()),
             guard: None,
             id,
             pat: pat(),
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index 8932200c5b7..64de5e92abf 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -1000,8 +1000,10 @@ impl EarlyLintPass for UnusedDocComment {
     }
 
     fn check_arm(&mut self, cx: &EarlyContext<'_>, arm: &ast::Arm) {
-        let arm_span = arm.pat.span.with_hi(arm.body.span.hi());
-        warn_if_doc(cx, arm_span, "match arms", &arm.attrs);
+        if let Some(body) = &arm.body {
+            let arm_span = arm.pat.span.with_hi(body.span.hi());
+            warn_if_doc(cx, arm_span, "match arms", &arm.attrs);
+        }
     }
 
     fn check_pat(&mut self, cx: &EarlyContext<'_>, pat: &ast::Pat) {
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 0a167b0893c..92deebb9d2c 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -1113,15 +1113,17 @@ impl EarlyLintPass for UnusedParens {
             }
             ExprKind::Match(ref _expr, ref arm) => {
                 for a in arm {
-                    self.check_unused_delims_expr(
-                        cx,
-                        &a.body,
-                        UnusedDelimsCtx::MatchArmExpr,
-                        false,
-                        None,
-                        None,
-                        true,
-                    );
+                    if let Some(body) = &a.body {
+                        self.check_unused_delims_expr(
+                            cx,
+                            body,
+                            UnusedDelimsCtx::MatchArmExpr,
+                            false,
+                            None,
+                            None,
+                            true,
+                        );
+                    }
                 }
             }
             _ => {}
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index b1b77305e4f..09c0dd78581 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -2899,127 +2899,155 @@ impl<'a> Parser<'a> {
         self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
             let lo = this.token.span;
             let (pat, guard) = this.parse_match_arm_pat_and_guard()?;
-            let arrow_span = this.token.span;
-            if let Err(mut err) = this.expect(&token::FatArrow) {
-                // We might have a `=>` -> `=` or `->` typo (issue #89396).
-                if TokenKind::FatArrow
-                    .similar_tokens()
-                    .is_some_and(|similar_tokens| similar_tokens.contains(&this.token.kind))
-                {
-                    err.span_suggestion(
-                        this.token.span,
-                        "use a fat arrow to start a match arm",
-                        "=>",
-                        Applicability::MachineApplicable,
-                    );
-                    if matches!(
-                        (&this.prev_token.kind, &this.token.kind),
-                        (token::DotDotEq, token::Gt)
-                    ) {
-                        // `error_inclusive_range_match_arrow` handles cases like `0..=> {}`,
-                        // so we suppress the error here
-                        err.delay_as_bug();
+
+            let span_before_body = this.prev_token.span;
+            let arm_body;
+            let is_fat_arrow = this.check(&token::FatArrow);
+            let is_almost_fat_arrow = TokenKind::FatArrow
+                .similar_tokens()
+                .is_some_and(|similar_tokens| similar_tokens.contains(&this.token.kind));
+            let mut result = if !is_fat_arrow && !is_almost_fat_arrow {
+                // A pattern without a body, allowed for never patterns.
+                arm_body = None;
+                this.expect_one_of(&[token::Comma], &[token::CloseDelim(Delimiter::Brace)])
+            } else {
+                if let Err(mut err) = this.expect(&token::FatArrow) {
+                    // We might have a `=>` -> `=` or `->` typo (issue #89396).
+                    if is_almost_fat_arrow {
+                        err.span_suggestion(
+                            this.token.span,
+                            "use a fat arrow to start a match arm",
+                            "=>",
+                            Applicability::MachineApplicable,
+                        );
+                        if matches!(
+                            (&this.prev_token.kind, &this.token.kind),
+                            (token::DotDotEq, token::Gt)
+                        ) {
+                            // `error_inclusive_range_match_arrow` handles cases like `0..=> {}`,
+                            // so we suppress the error here
+                            err.delay_as_bug();
+                        } else {
+                            err.emit();
+                        }
+                        this.bump();
                     } else {
-                        err.emit();
+                        return Err(err);
                     }
-                    this.bump();
-                } else {
-                    return Err(err);
                 }
-            }
-            let arm_start_span = this.token.span;
+                let arrow_span = this.prev_token.span;
+                let arm_start_span = this.token.span;
 
-            let expr = this.parse_expr_res(Restrictions::STMT_EXPR, None).map_err(|mut err| {
-                err.span_label(arrow_span, "while parsing the `match` arm starting here");
-                err
-            })?;
+                let expr =
+                    this.parse_expr_res(Restrictions::STMT_EXPR, None).map_err(|mut err| {
+                        err.span_label(arrow_span, "while parsing the `match` arm starting here");
+                        err
+                    })?;
 
-            let require_comma = classify::expr_requires_semi_to_be_stmt(&expr)
-                && this.token != token::CloseDelim(Delimiter::Brace);
-
-            let hi = this.prev_token.span;
-
-            if require_comma {
-                let sm = this.sess.source_map();
-                if let Some(body) = this.parse_arm_body_missing_braces(&expr, arrow_span) {
-                    let span = body.span;
-                    return Ok((
-                        ast::Arm {
-                            attrs,
-                            pat,
-                            guard,
-                            body,
-                            span,
-                            id: DUMMY_NODE_ID,
-                            is_placeholder: false,
-                        },
-                        TrailingToken::None,
-                    ));
-                }
-                this.expect_one_of(&[token::Comma], &[token::CloseDelim(Delimiter::Brace)])
-                    .or_else(|mut err| {
-                        if this.token == token::FatArrow {
-                            if let Ok(expr_lines) = sm.span_to_lines(expr.span)
-                                && let Ok(arm_start_lines) = sm.span_to_lines(arm_start_span)
-                                && arm_start_lines.lines[0].end_col == expr_lines.lines[0].end_col
-                                && expr_lines.lines.len() == 2
-                            {
-                                // We check whether there's any trailing code in the parse span,
-                                // if there isn't, we very likely have the following:
-                                //
-                                // X |     &Y => "y"
-                                //   |        --    - missing comma
-                                //   |        |
-                                //   |        arrow_span
-                                // X |     &X => "x"
-                                //   |      - ^^ self.token.span
-                                //   |      |
-                                //   |      parsed until here as `"y" & X`
-                                err.span_suggestion_short(
-                                    arm_start_span.shrink_to_hi(),
-                                    "missing a comma here to end this `match` arm",
-                                    ",",
-                                    Applicability::MachineApplicable,
+                let require_comma = classify::expr_requires_semi_to_be_stmt(&expr)
+                    && this.token != token::CloseDelim(Delimiter::Brace);
+
+                if !require_comma {
+                    arm_body = Some(expr);
+                    this.eat(&token::Comma);
+                    Ok(false)
+                } else if let Some(body) = this.parse_arm_body_missing_braces(&expr, arrow_span) {
+                    arm_body = Some(body);
+                    Ok(true)
+                } else {
+                    let expr_span = expr.span;
+                    arm_body = Some(expr);
+                    this.expect_one_of(&[token::Comma], &[token::CloseDelim(Delimiter::Brace)])
+                        .map_err(|mut err| {
+                            if this.token == token::FatArrow {
+                                let sm = this.sess.source_map();
+                                if let Ok(expr_lines) = sm.span_to_lines(expr_span)
+                                    && let Ok(arm_start_lines) = sm.span_to_lines(arm_start_span)
+                                    && arm_start_lines.lines[0].end_col
+                                        == expr_lines.lines[0].end_col
+                                    && expr_lines.lines.len() == 2
+                                {
+                                    // We check whether there's any trailing code in the parse span,
+                                    // if there isn't, we very likely have the following:
+                                    //
+                                    // X |     &Y => "y"
+                                    //   |        --    - missing comma
+                                    //   |        |
+                                    //   |        arrow_span
+                                    // X |     &X => "x"
+                                    //   |      - ^^ self.token.span
+                                    //   |      |
+                                    //   |      parsed until here as `"y" & X`
+                                    err.span_suggestion_short(
+                                        arm_start_span.shrink_to_hi(),
+                                        "missing a comma here to end this `match` arm",
+                                        ",",
+                                        Applicability::MachineApplicable,
+                                    );
+                                }
+                            } else {
+                                err.span_label(
+                                    arrow_span,
+                                    "while parsing the `match` arm starting here",
                                 );
-                                return Err(err);
-                            }
-                        } else {
-                            // FIXME(compiler-errors): We could also recover `; PAT =>` here
-
-                            // Try to parse a following `PAT =>`, if successful
-                            // then we should recover.
-                            let mut snapshot = this.create_snapshot_for_diagnostic();
-                            let pattern_follows = snapshot
-                                .parse_pat_allow_top_alt(
-                                    None,
-                                    RecoverComma::Yes,
-                                    RecoverColon::Yes,
-                                    CommaRecoveryMode::EitherTupleOrPipe,
-                                )
-                                .map_err(|err| err.cancel())
-                                .is_ok();
-                            if pattern_follows && snapshot.check(&TokenKind::FatArrow) {
-                                err.cancel();
-                                this.sess.emit_err(errors::MissingCommaAfterMatchArm {
-                                    span: hi.shrink_to_hi(),
-                                });
-                                return Ok(true);
                             }
-                        }
-                        err.span_label(arrow_span, "while parsing the `match` arm starting here");
-                        Err(err)
-                    })?;
-            } else {
-                this.eat(&token::Comma);
+                            err
+                        })
+                }
+            };
+
+            let hi_span = arm_body.as_ref().map_or(span_before_body, |body| body.span);
+            let arm_span = lo.to(hi_span);
+
+            // We want to recover:
+            // X |     Some(_) => foo()
+            //   |                     - missing comma
+            // X |     None => "x"
+            //   |     ^^^^ self.token.span
+            // as well as:
+            // X |     Some(!)
+            //   |            - missing comma
+            // X |     None => "x"
+            //   |     ^^^^ self.token.span
+            // But we musn't recover
+            // X |     pat[0] => {}
+            //   |        ^ self.token.span
+            let recover_missing_comma = arm_body.is_some() || pat.could_be_never_pattern();
+            if recover_missing_comma {
+                result = result.or_else(|err| {
+                    // FIXME(compiler-errors): We could also recover `; PAT =>` here
+
+                    // Try to parse a following `PAT =>`, if successful
+                    // then we should recover.
+                    let mut snapshot = this.create_snapshot_for_diagnostic();
+                    let pattern_follows = snapshot
+                        .parse_pat_allow_top_alt(
+                            None,
+                            RecoverComma::Yes,
+                            RecoverColon::Yes,
+                            CommaRecoveryMode::EitherTupleOrPipe,
+                        )
+                        .map_err(|err| err.cancel())
+                        .is_ok();
+                    if pattern_follows && snapshot.check(&TokenKind::FatArrow) {
+                        err.cancel();
+                        this.sess.emit_err(errors::MissingCommaAfterMatchArm {
+                            span: arm_span.shrink_to_hi(),
+                        });
+                        return Ok(true);
+                    }
+                    Err(err)
+                });
             }
+            result?;
 
             Ok((
                 ast::Arm {
                     attrs,
                     pat,
                     guard,
-                    body: expr,
-                    span: lo.to(hi),
+                    body: arm_body,
+                    span: arm_span,
                     id: DUMMY_NODE_ID,
                     is_placeholder: false,
                 },
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index 3d1d1ec8108..9b8a34cf0cc 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -154,7 +154,7 @@ impl<'a> Parser<'a> {
             }
             Err(err) => return Err(err),
         };
-        if rc == RecoverComma::Yes {
+        if rc == RecoverComma::Yes && !first_pat.could_be_never_pattern() {
             self.maybe_recover_unexpected_comma(
                 first_pat.span,
                 matches!(first_pat.kind, PatKind::MacCall(_)),
@@ -200,7 +200,7 @@ impl<'a> Parser<'a> {
                 err.span_label(lo, WHILE_PARSING_OR_MSG);
                 err
             })?;
-            if rc == RecoverComma::Yes {
+            if rc == RecoverComma::Yes && !pat.could_be_never_pattern() {
                 self.maybe_recover_unexpected_comma(pat.span, false, rt)?;
             }
             pats.push(pat);
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 75a0541b89b..5a139244640 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -3294,7 +3294,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
         self.with_rib(ValueNS, RibKind::Normal, |this| {
             this.resolve_pattern_top(&arm.pat, PatternSource::Match);
             walk_list!(this, visit_expr, &arm.guard);
-            this.visit_expr(&arm.body);
+            walk_list!(this, visit_expr, &arm.body);
         });
     }
 
diff --git a/src/tools/clippy/clippy_lints/src/non_expressive_names.rs b/src/tools/clippy/clippy_lints/src/non_expressive_names.rs
index 649a23565a9..107f488484d 100644
--- a/src/tools/clippy/clippy_lints/src/non_expressive_names.rs
+++ b/src/tools/clippy/clippy_lints/src/non_expressive_names.rs
@@ -341,7 +341,9 @@ impl<'a, 'tcx> Visitor<'tcx> for SimilarNamesLocalVisitor<'a, 'tcx> {
 
         self.apply(|this| {
             SimilarNamesNameVisitor(this).visit_pat(&arm.pat);
-            this.apply(|this| walk_expr(this, &arm.body));
+            if let Some(body) = &arm.body {
+                this.apply(|this| walk_expr(this, body));
+            }
         });
 
         self.check_single_char_names();
diff --git a/src/tools/clippy/clippy_lints/src/redundant_else.rs b/src/tools/clippy/clippy_lints/src/redundant_else.rs
index 221aa317e5d..530fc664657 100644
--- a/src/tools/clippy/clippy_lints/src/redundant_else.rs
+++ b/src/tools/clippy/clippy_lints/src/redundant_else.rs
@@ -105,7 +105,9 @@ impl<'ast> Visitor<'ast> for BreakVisitor {
     fn visit_expr(&mut self, expr: &'ast Expr) {
         self.is_break = match expr.kind {
             ExprKind::Break(..) | ExprKind::Continue(..) | ExprKind::Ret(..) => true,
-            ExprKind::Match(_, ref arms) => arms.iter().all(|arm| self.check_expr(&arm.body)),
+            ExprKind::Match(_, ref arms) => arms.iter().all(|arm|
+                arm.body.is_none() || arm.body.as_deref().is_some_and(|body| self.check_expr(body))
+            ),
             ExprKind::If(_, ref then, Some(ref els)) => self.check_block(then) && self.check_expr(els),
             ExprKind::If(_, _, None)
             // ignore loops for simplicity
diff --git a/src/tools/clippy/clippy_utils/src/ast_utils.rs b/src/tools/clippy/clippy_utils/src/ast_utils.rs
index a2c61e07b70..b2cff164661 100644
--- a/src/tools/clippy/clippy_utils/src/ast_utils.rs
+++ b/src/tools/clippy/clippy_utils/src/ast_utils.rs
@@ -236,7 +236,7 @@ pub fn eq_field(l: &ExprField, r: &ExprField) -> bool {
 pub fn eq_arm(l: &Arm, r: &Arm) -> bool {
     l.is_placeholder == r.is_placeholder
         && eq_pat(&l.pat, &r.pat)
-        && eq_expr(&l.body, &r.body)
+        && eq_expr_opt(&l.body, &r.body)
         && eq_expr_opt(&l.guard, &r.guard)
         && over(&l.attrs, &r.attrs, eq_attr)
 }
diff --git a/src/tools/rustfmt/src/matches.rs b/src/tools/rustfmt/src/matches.rs
index 95b0ed16db8..ef509b56837 100644
--- a/src/tools/rustfmt/src/matches.rs
+++ b/src/tools/rustfmt/src/matches.rs
@@ -223,7 +223,7 @@ fn rewrite_match_arm(
 ) -> Option<String> {
     let (missing_span, attrs_str) = if !arm.attrs.is_empty() {
         if contains_skip(&arm.attrs) {
-            let (_, body) = flatten_arm_body(context, &arm.body, None);
+            let (_, body) = flatten_arm_body(context, arm.body.as_deref()?, None);
             // `arm.span()` does not include trailing comma, add it manually.
             return Some(format!(
                 "{}{}",
@@ -246,7 +246,7 @@ fn rewrite_match_arm(
     };
 
     // Patterns
-    let pat_shape = match &arm.body.kind {
+    let pat_shape = match &arm.body.as_ref()?.kind {
         ast::ExprKind::Block(_, Some(label)) => {
             // Some block with a label ` => 'label: {`
             // 7 = ` => : {`
@@ -280,10 +280,10 @@ fn rewrite_match_arm(
         false,
     )?;
 
-    let arrow_span = mk_sp(arm.pat.span.hi(), arm.body.span().lo());
+    let arrow_span = mk_sp(arm.pat.span.hi(), arm.body.as_ref()?.span().lo());
     rewrite_match_body(
         context,
-        &arm.body,
+        arm.body.as_ref()?,
         &lhs_str,
         shape,
         guard_str.contains('\n'),
diff --git a/src/tools/rustfmt/src/spanned.rs b/src/tools/rustfmt/src/spanned.rs
index 2136cfeae1a..5960b144499 100644
--- a/src/tools/rustfmt/src/spanned.rs
+++ b/src/tools/rustfmt/src/spanned.rs
@@ -97,7 +97,12 @@ impl Spanned for ast::Arm {
         } else {
             self.attrs[0].span.lo()
         };
-        span_with_attrs_lo_hi!(self, lo, self.body.span.hi())
+        let hi = if let Some(body) = &self.body {
+            body.span.hi()
+        } else {
+            self.pat.span.hi()
+        };
+        span_with_attrs_lo_hi!(self, lo, hi)
     }
 }
 
diff --git a/tests/ui/half-open-range-patterns/range_pat_interactions1.rs b/tests/ui/half-open-range-patterns/range_pat_interactions1.rs
index 55353999b67..9ffc2190d20 100644
--- a/tests/ui/half-open-range-patterns/range_pat_interactions1.rs
+++ b/tests/ui/half-open-range-patterns/range_pat_interactions1.rs
@@ -17,7 +17,7 @@ fn main() {
         }
         match x as i32 {
             0..5+1 => errors_only.push(x),
-            //~^ error: expected one of `=>`, `if`, or `|`, found `+`
+            //~^ error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `+`
             1 | -3..0 => first_or.push(x),
             y @ (0..5 | 6) => or_two.push(y),
             y @ 0..const { 5 + 1 } => assert_eq!(y, 5),
diff --git a/tests/ui/half-open-range-patterns/range_pat_interactions1.stderr b/tests/ui/half-open-range-patterns/range_pat_interactions1.stderr
index 19ebcaf0f36..05235c9b922 100644
--- a/tests/ui/half-open-range-patterns/range_pat_interactions1.stderr
+++ b/tests/ui/half-open-range-patterns/range_pat_interactions1.stderr
@@ -1,8 +1,8 @@
-error: expected one of `=>`, `if`, or `|`, found `+`
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `+`
   --> $DIR/range_pat_interactions1.rs:19:17
    |
 LL |             0..5+1 => errors_only.push(x),
-   |                 ^ expected one of `=>`, `if`, or `|`
+   |                 ^ expected one of `,`, `=>`, `if`, `|`, or `}`
 
 error[E0408]: variable `n` is not bound in all patterns
   --> $DIR/range_pat_interactions1.rs:10:25
diff --git a/tests/ui/half-open-range-patterns/range_pat_interactions2.rs b/tests/ui/half-open-range-patterns/range_pat_interactions2.rs
index 4615ebd688a..b212bfbe093 100644
--- a/tests/ui/half-open-range-patterns/range_pat_interactions2.rs
+++ b/tests/ui/half-open-range-patterns/range_pat_interactions2.rs
@@ -9,7 +9,7 @@ fn main() {
         match x as i32 {
             0..=(5+1) => errors_only.push(x),
             //~^ error: inclusive range with no end
-            //~| error: expected one of `=>`, `if`, or `|`, found `(`
+            //~| error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `(`
             1 | -3..0 => first_or.push(x),
             y @ (0..5 | 6) => or_two.push(y),
             y @ 0..const { 5 + 1 } => assert_eq!(y, 5),
diff --git a/tests/ui/half-open-range-patterns/range_pat_interactions2.stderr b/tests/ui/half-open-range-patterns/range_pat_interactions2.stderr
index 13a5542a474..0129f927e34 100644
--- a/tests/ui/half-open-range-patterns/range_pat_interactions2.stderr
+++ b/tests/ui/half-open-range-patterns/range_pat_interactions2.stderr
@@ -6,11 +6,11 @@ LL |             0..=(5+1) => errors_only.push(x),
    |
    = note: inclusive ranges must be bounded at the end (`..=b` or `a..=b`)
 
-error: expected one of `=>`, `if`, or `|`, found `(`
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `(`
   --> $DIR/range_pat_interactions2.rs:10:17
    |
 LL |             0..=(5+1) => errors_only.push(x),
-   |                 ^ expected one of `=>`, `if`, or `|`
+   |                 ^ expected one of `,`, `=>`, `if`, `|`, or `}`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/never_patterns/check.rs b/tests/ui/never_patterns/check.rs
index bcc3a760c10..9b02fc7996d 100644
--- a/tests/ui/never_patterns/check.rs
+++ b/tests/ui/never_patterns/check.rs
@@ -15,11 +15,12 @@ fn no_arms_or_guards(x: Void) {
         None => {}
     }
     match None::<Void> {
+        //~^ ERROR non-exhaustive
         Some(!) if true,
-        //~^ ERROR expected one of
         None => {}
     }
     match None::<Void> {
+        //~^ ERROR non-exhaustive
         Some(!) if true => {}
         None => {}
     }
diff --git a/tests/ui/never_patterns/check.stderr b/tests/ui/never_patterns/check.stderr
index d7cdd64840f..945812225c4 100644
--- a/tests/ui/never_patterns/check.stderr
+++ b/tests/ui/never_patterns/check.stderr
@@ -1,8 +1,39 @@
-error: expected one of `.`, `=>`, `?`, or an operator, found `,`
-  --> $DIR/check.rs:18:24
+error[E0004]: non-exhaustive patterns: `Some(_)` not covered
+  --> $DIR/check.rs:17:11
+   |
+LL |     match None::<Void> {
+   |           ^^^^^^^^^^^^ pattern `Some(_)` not covered
+   |
+note: `Option<Void>` defined here
+  --> $SRC_DIR/core/src/option.rs:LL:COL
+  ::: $SRC_DIR/core/src/option.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Option<Void>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         None => {},
+LL +         Some(_) => todo!()
+   |
+
+error[E0004]: non-exhaustive patterns: `Some(_)` not covered
+  --> $DIR/check.rs:22:11
+   |
+LL |     match None::<Void> {
+   |           ^^^^^^^^^^^^ pattern `Some(_)` not covered
+   |
+note: `Option<Void>` defined here
+  --> $SRC_DIR/core/src/option.rs:LL:COL
+  ::: $SRC_DIR/core/src/option.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Option<Void>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         None => {},
+LL +         Some(_) => todo!()
    |
-LL |         Some(!) if true,
-   |                        ^ expected one of `.`, `=>`, `?`, or an operator
 
-error: aborting due to 1 previous error
+error: aborting due to 2 previous errors
 
+For more information about this error, try `rustc --explain E0004`.
diff --git a/tests/ui/never_patterns/parse.rs b/tests/ui/never_patterns/parse.rs
new file mode 100644
index 00000000000..850416d723a
--- /dev/null
+++ b/tests/ui/never_patterns/parse.rs
@@ -0,0 +1,68 @@
+#![feature(never_patterns)]
+#![allow(incomplete_features)]
+
+enum Void {}
+
+fn main() {}
+
+macro_rules! never {
+    () => { ! }
+}
+
+fn parse(x: Void) {
+    match None::<Void> {
+        None => {}
+        Some(!),
+    }
+    match None::<Void> {
+        Some(!),
+        None => {}
+    }
+    match None::<Void> {
+        None => {}
+        Some(!)
+    }
+    match None::<Void> {
+        Some(!)
+        //~^ ERROR expected `,` following `match` arm
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true
+        //~^ ERROR expected `,` following `match` arm
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true,
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) <=
+        //~^ ERROR expected one of
+    }
+    match x {
+        never!(),
+    }
+    match x {
+        never!() if true,
+    }
+    match x {
+        never!()
+    }
+    match &x {
+        &never!(),
+    }
+    match None::<Void> {
+        Some(never!()),
+        None => {}
+    }
+    match x { ! }
+    match &x { &! }
+
+    let res: Result<bool, Void> = Ok(false);
+    let Ok(_) = res;
+    let Ok(_) | Err(!) = &res; // Disallowed; see #82048.
+    //~^ ERROR top-level or-patterns are not allowed in `let` bindings
+    let (Ok(_) | Err(!)) = &res;
+    let (Ok(_) | Err(&!)) = res.as_ref();
+}
diff --git a/tests/ui/never_patterns/parse.stderr b/tests/ui/never_patterns/parse.stderr
new file mode 100644
index 00000000000..7ea33540c5e
--- /dev/null
+++ b/tests/ui/never_patterns/parse.stderr
@@ -0,0 +1,26 @@
+error: expected `,` following `match` arm
+  --> $DIR/parse.rs:26:16
+   |
+LL |         Some(!)
+   |                ^ help: missing a comma here to end this `match` arm: `,`
+
+error: expected `,` following `match` arm
+  --> $DIR/parse.rs:31:24
+   |
+LL |         Some(!) if true
+   |                        ^ help: missing a comma here to end this `match` arm: `,`
+
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `<=`
+  --> $DIR/parse.rs:40:17
+   |
+LL |         Some(!) <=
+   |                 ^^ expected one of `,`, `=>`, `if`, `|`, or `}`
+
+error: top-level or-patterns are not allowed in `let` bindings
+  --> $DIR/parse.rs:64:9
+   |
+LL |     let Ok(_) | Err(!) = &res; // Disallowed; see #82048.
+   |         ^^^^^^^^^^^^^^ help: wrap the pattern in parentheses: `(Ok(_) | Err(!))`
+
+error: aborting due to 4 previous errors
+
diff --git a/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.rs b/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.rs
index d1950087c4c..2c402e4c65e 100644
--- a/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.rs
+++ b/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.rs
@@ -84,15 +84,15 @@ fn main() {}
 
 #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] 10 => () } }
 //~^ ERROR inclusive range with no end
-//~| ERROR expected one of `=>`, `if`, or `|`, found `#`
+//~| ERROR expected one of `,`, `=>`, `if`, `|`, or `}`, found `#`
 #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] -10 => () } }
 //~^ ERROR inclusive range with no end
-//~| ERROR expected one of `=>`, `if`, or `|`, found `#`
+//~| ERROR expected one of `,`, `=>`, `if`, `|`, or `}`, found `#`
 #[cfg(FALSE)] fn e() { match 0 { 0..=-#[attr] 10 => () } }
 //~^ ERROR unexpected token: `#`
 #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] FOO => () } }
 //~^ ERROR inclusive range with no end
-//~| ERROR expected one of `=>`, `if`, or `|`, found `#`
+//~| ERROR expected one of `,`, `=>`, `if`, `|`, or `}`, found `#`
 
 #[cfg(FALSE)] fn e() { let _ = x.#![attr]foo(); }
 //~^ ERROR unexpected token: `#`
diff --git a/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.stderr b/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.stderr
index e46c591080d..a0e95c5c1ed 100644
--- a/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.stderr
+++ b/tests/ui/parser/attribute/attr-stmt-expr-attr-bad.stderr
@@ -365,11 +365,11 @@ LL | #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] 10 => () } }
    |
    = note: inclusive ranges must be bounded at the end (`..=b` or `a..=b`)
 
-error: expected one of `=>`, `if`, or `|`, found `#`
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `#`
   --> $DIR/attr-stmt-expr-attr-bad.rs:85:38
    |
 LL | #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] 10 => () } }
-   |                                      ^ expected one of `=>`, `if`, or `|`
+   |                                      ^ expected one of `,`, `=>`, `if`, `|`, or `}`
 
 error[E0586]: inclusive range with no end
   --> $DIR/attr-stmt-expr-attr-bad.rs:88:35
@@ -379,11 +379,11 @@ LL | #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] -10 => () } }
    |
    = note: inclusive ranges must be bounded at the end (`..=b` or `a..=b`)
 
-error: expected one of `=>`, `if`, or `|`, found `#`
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `#`
   --> $DIR/attr-stmt-expr-attr-bad.rs:88:38
    |
 LL | #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] -10 => () } }
-   |                                      ^ expected one of `=>`, `if`, or `|`
+   |                                      ^ expected one of `,`, `=>`, `if`, `|`, or `}`
 
 error: unexpected token: `#`
   --> $DIR/attr-stmt-expr-attr-bad.rs:91:39
@@ -399,11 +399,11 @@ LL | #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] FOO => () } }
    |
    = note: inclusive ranges must be bounded at the end (`..=b` or `a..=b`)
 
-error: expected one of `=>`, `if`, or `|`, found `#`
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `#`
   --> $DIR/attr-stmt-expr-attr-bad.rs:93:38
    |
 LL | #[cfg(FALSE)] fn e() { match 0 { 0..=#[attr] FOO => () } }
-   |                                      ^ expected one of `=>`, `if`, or `|`
+   |                                      ^ expected one of `,`, `=>`, `if`, `|`, or `}`
 
 error: unexpected token: `#`
   --> $DIR/attr-stmt-expr-attr-bad.rs:97:34
diff --git a/tests/ui/parser/issues/issue-24375.rs b/tests/ui/parser/issues/issue-24375.rs
index 1d128d33e4f..8d1bc579e7b 100644
--- a/tests/ui/parser/issues/issue-24375.rs
+++ b/tests/ui/parser/issues/issue-24375.rs
@@ -3,7 +3,7 @@ static tmp : [&'static str; 2]  = ["hello", "he"];
 fn main() {
     let z = "hello";
     match z {
-        tmp[0] => {} //~ ERROR expected one of `=>`, `@`, `if`, or `|`, found `[`
+        tmp[0] => {} //~ ERROR expected one of `,`, `=>`, `@`, `if`, `|`, or `}`, found `[`
         _ => {}
     }
 }
diff --git a/tests/ui/parser/issues/issue-24375.stderr b/tests/ui/parser/issues/issue-24375.stderr
index bb1e19e9e6d..2b980a5520f 100644
--- a/tests/ui/parser/issues/issue-24375.stderr
+++ b/tests/ui/parser/issues/issue-24375.stderr
@@ -1,8 +1,8 @@
-error: expected one of `=>`, `@`, `if`, or `|`, found `[`
+error: expected one of `,`, `=>`, `@`, `if`, `|`, or `}`, found `[`
   --> $DIR/issue-24375.rs:6:12
    |
 LL |         tmp[0] => {}
-   |            ^ expected one of `=>`, `@`, `if`, or `|`
+   |            ^ expected one of `,`, `=>`, `@`, `if`, `|`, or `}`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/parser/macro/macro-expand-to-match-arm.rs b/tests/ui/parser/macro/macro-expand-to-match-arm.rs
index 39d1d065ed9..972ca61cc84 100644
--- a/tests/ui/parser/macro/macro-expand-to-match-arm.rs
+++ b/tests/ui/parser/macro/macro-expand-to-match-arm.rs
@@ -1,6 +1,8 @@
 macro_rules! arm {
     ($pattern:pat => $block:block) => {
         $pattern => $block
+        //~^ ERROR macro expansion ignores token `=>` and any following
+        //~| NOTE the usage of `arm!` is likely invalid in pattern context
     };
 }
 
@@ -9,9 +11,7 @@ fn main() {
     match x {
         Some(1) => {},
         arm!(None => {}),
-        //~^ NOTE macros cannot expand to match arms
-        //~| ERROR unexpected `,` in pattern
-        // doesn't recover
+        //~^ NOTE caused by the macro expansion here
         Some(2) => {},
         _ => {},
     };
diff --git a/tests/ui/parser/macro/macro-expand-to-match-arm.stderr b/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
index 1b34d2d12b2..a62109c5050 100644
--- a/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
+++ b/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
@@ -1,10 +1,13 @@
-error: unexpected `,` in pattern
-  --> $DIR/macro-expand-to-match-arm.rs:11:25
+error: macro expansion ignores token `=>` and any following
+  --> $DIR/macro-expand-to-match-arm.rs:3:18
    |
+LL |         $pattern => $block
+   |                  ^^
+...
 LL |         arm!(None => {}),
-   |                         ^
+   |         ---------------- caused by the macro expansion here
    |
-   = note: macros cannot expand to match arms
+   = note: the usage of `arm!` is likely invalid in pattern context
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/parser/match-arm-without-body.rs b/tests/ui/parser/match-arm-without-body.rs
index 5f009c7a355..c33bd0c3031 100644
--- a/tests/ui/parser/match-arm-without-body.rs
+++ b/tests/ui/parser/match-arm-without-body.rs
@@ -6,7 +6,6 @@ fn main() {
     match Some(false) {
         Some(_)
     }
-    //~^ ERROR expected one of
     match Some(false) {
         Some(_)
         _ => {}
@@ -28,7 +27,6 @@ fn main() {
     match Some(false) {
         Some(_) if true
     }
-    //~^ ERROR expected one of
     match Some(false) {
         Some(_) if true
         _ => {}
@@ -36,33 +34,28 @@ fn main() {
     }
     match Some(false) {
         Some(_) if true,
-        //~^ ERROR expected one of
     }
     match Some(false) {
         Some(_) if true,
-        //~^ ERROR expected one of
         _ => {}
     }
     match Some(false) {
         pat!()
     }
-    //~^ ERROR expected one of
     match Some(false) {
         pat!(),
-        //~^ ERROR unexpected `,` in pattern
     }
     match Some(false) {
         pat!() if true,
-        //~^ ERROR expected one of
     }
     match Some(false) {
         pat!()
+        //~^ ERROR expected `,` following `match` arm
+        //~| HELP missing a comma here
         _ => {}
-        //~^ ERROR expected one of
     }
     match Some(false) {
         pat!(),
-        //~^ ERROR unexpected `,` in pattern
         _ => {}
     }
 }
diff --git a/tests/ui/parser/match-arm-without-body.stderr b/tests/ui/parser/match-arm-without-body.stderr
index 210007628db..77fd9176633 100644
--- a/tests/ui/parser/match-arm-without-body.stderr
+++ b/tests/ui/parser/match-arm-without-body.stderr
@@ -1,21 +1,13 @@
-error: expected one of `=>`, `if`, or `|`, found `}`
-  --> $DIR/match-arm-without-body.rs:8:5
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found reserved identifier `_`
+  --> $DIR/match-arm-without-body.rs:11:9
    |
 LL |         Some(_)
-   |                - expected one of `=>`, `if`, or `|`
-LL |     }
-   |     ^ unexpected token
-
-error: expected one of `=>`, `if`, or `|`, found reserved identifier `_`
-  --> $DIR/match-arm-without-body.rs:12:9
-   |
-LL |         Some(_)
-   |                - expected one of `=>`, `if`, or `|`
+   |                - expected one of `,`, `=>`, `if`, `|`, or `}`
 LL |         _ => {}
    |         ^ unexpected token
 
 error: unexpected `,` in pattern
-  --> $DIR/match-arm-without-body.rs:16:16
+  --> $DIR/match-arm-without-body.rs:15:16
    |
 LL |         Some(_),
    |                ^
@@ -30,7 +22,7 @@ LL |         Some(_) |
    |
 
 error: unexpected `,` in pattern
-  --> $DIR/match-arm-without-body.rs:22:16
+  --> $DIR/match-arm-without-body.rs:21:16
    |
 LL |         Some(_),
    |                ^
@@ -52,71 +44,19 @@ LL +
 LL ~         _ => {}
    |
 
-error: expected one of `.`, `=>`, `?`, or an operator, found `}`
-  --> $DIR/match-arm-without-body.rs:30:5
+error: expected one of `,`, `.`, `=>`, `?`, `}`, or an operator, found reserved identifier `_`
+  --> $DIR/match-arm-without-body.rs:32:9
    |
 LL |         Some(_) if true
-   |                        - expected one of `.`, `=>`, `?`, or an operator
-LL |     }
-   |     ^ unexpected token
-
-error: expected one of `.`, `=>`, `?`, or an operator, found reserved identifier `_`
-  --> $DIR/match-arm-without-body.rs:34:9
-   |
-LL |         Some(_) if true
-   |                        - expected one of `.`, `=>`, `?`, or an operator
+   |                        - expected one of `,`, `.`, `=>`, `?`, `}`, or an operator
 LL |         _ => {}
    |         ^ unexpected token
 
-error: expected one of `.`, `=>`, `?`, or an operator, found `,`
-  --> $DIR/match-arm-without-body.rs:38:24
-   |
-LL |         Some(_) if true,
-   |                        ^ expected one of `.`, `=>`, `?`, or an operator
-
-error: expected one of `.`, `=>`, `?`, or an operator, found `,`
-  --> $DIR/match-arm-without-body.rs:42:24
-   |
-LL |         Some(_) if true,
-   |                        ^ expected one of `.`, `=>`, `?`, or an operator
-
-error: expected one of `=>`, `if`, or `|`, found `}`
-  --> $DIR/match-arm-without-body.rs:48:5
+error: expected `,` following `match` arm
+  --> $DIR/match-arm-without-body.rs:52:15
    |
 LL |         pat!()
-   |               - expected one of `=>`, `if`, or `|`
-LL |     }
-   |     ^ unexpected token
-
-error: unexpected `,` in pattern
-  --> $DIR/match-arm-without-body.rs:51:15
-   |
-LL |         pat!(),
-   |               ^
-   |
-   = note: macros cannot expand to match arms
-
-error: expected one of `.`, `=>`, `?`, or an operator, found `,`
-  --> $DIR/match-arm-without-body.rs:55:23
-   |
-LL |         pat!() if true,
-   |                       ^ expected one of `.`, `=>`, `?`, or an operator
-
-error: expected one of `=>`, `if`, or `|`, found reserved identifier `_`
-  --> $DIR/match-arm-without-body.rs:60:9
-   |
-LL |         pat!()
-   |               - expected one of `=>`, `if`, or `|`
-LL |         _ => {}
-   |         ^ unexpected token
-
-error: unexpected `,` in pattern
-  --> $DIR/match-arm-without-body.rs:64:15
-   |
-LL |         pat!(),
-   |               ^
-   |
-   = note: macros cannot expand to match arms
+   |               ^ help: missing a comma here to end this `match` arm: `,`
 
-error: aborting due to 13 previous errors
+error: aborting due to 5 previous errors
 
diff --git a/tests/ui/parser/pat-lt-bracket-1.rs b/tests/ui/parser/pat-lt-bracket-1.rs
index 2e2001434f2..33da15adb9e 100644
--- a/tests/ui/parser/pat-lt-bracket-1.rs
+++ b/tests/ui/parser/pat-lt-bracket-1.rs
@@ -1,7 +1,7 @@
 fn main() {
   match 42 {
     x < 7 => (),
-   //~^ error: expected one of `=>`, `@`, `if`, or `|`, found `<`
+   //~^ error: expected one of `,`, `=>`, `@`, `if`, `|`, or `}`, found `<`
     _ => ()
   }
 }
diff --git a/tests/ui/parser/pat-lt-bracket-1.stderr b/tests/ui/parser/pat-lt-bracket-1.stderr
index 14e679bbee0..f39487052ad 100644
--- a/tests/ui/parser/pat-lt-bracket-1.stderr
+++ b/tests/ui/parser/pat-lt-bracket-1.stderr
@@ -1,8 +1,8 @@
-error: expected one of `=>`, `@`, `if`, or `|`, found `<`
+error: expected one of `,`, `=>`, `@`, `if`, `|`, or `}`, found `<`
   --> $DIR/pat-lt-bracket-1.rs:3:7
    |
 LL |     x < 7 => (),
-   |       ^ expected one of `=>`, `@`, `if`, or `|`
+   |       ^ expected one of `,`, `=>`, `@`, `if`, `|`, or `}`
 
 error: aborting due to 1 previous error
 
-- 
cgit 1.4.1-3-g733a5


From 0bfebc6105ea882d7048057718b2e34d09a5d17e Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 27 Nov 2023 00:50:51 +0100
Subject: Detect attempts to expand a macro to a match arm again

Because a macro invocation can expand to a never pattern, we can't rule
out a `arm!(),` arm at parse time. Instead we detect that case at
expansion time, if the macro tries to output a pattern followed by `=>`.
---
 compiler/rustc_expand/messages.ftl                 |  2 ++
 compiler/rustc_expand/src/errors.rs                |  2 ++
 compiler/rustc_expand/src/expand.rs                |  3 ++
 compiler/rustc_parse/messages.ftl                  |  2 --
 compiler/rustc_parse/src/parser/diagnostics.rs     | 37 ++++++++++------------
 compiler/rustc_parse/src/parser/pat.rs             |  8 ++---
 tests/ui/parser/macro/macro-expand-to-match-arm.rs |  1 +
 .../parser/macro/macro-expand-to-match-arm.stderr  |  1 +
 8 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/compiler/rustc_expand/messages.ftl b/compiler/rustc_expand/messages.ftl
index 8b93829623d..fc3f7b1d749 100644
--- a/compiler/rustc_expand/messages.ftl
+++ b/compiler/rustc_expand/messages.ftl
@@ -71,6 +71,8 @@ expand_macro_const_stability =
     .label = invalid const stability attribute
     .label2 = const stability attribute affects this macro
 
+expand_macro_expands_to_match_arm = macros cannot expand to match arms
+
 expand_malformed_feature_attribute =
     malformed `feature` attribute input
     .expected = expected just one word
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index d86632c47fc..6e919a8fa9f 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -304,6 +304,8 @@ pub(crate) struct IncompleteParse<'a> {
     pub label_span: Span,
     pub macro_path: &'a ast::Path,
     pub kind_name: &'a str,
+    #[note(expand_macro_expands_to_match_arm)]
+    pub expands_to_match_arm: Option<()>,
 
     #[suggestion(
         expand_suggestion_add_semi,
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index 1b51d80fb38..e2d2bc03280 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -955,12 +955,15 @@ pub fn ensure_complete_parse<'a>(
             _ => None,
         };
 
+        let expands_to_match_arm = kind_name == "pattern" && parser.token == token::FatArrow;
+
         parser.sess.emit_err(IncompleteParse {
             span: def_site_span,
             token,
             label_span: span,
             macro_path,
             kind_name,
+            expands_to_match_arm: expands_to_match_arm.then_some(()),
             add_semicolon,
         });
     }
diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
index 1c3c433d8b7..363b8f4bfb9 100644
--- a/compiler/rustc_parse/messages.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -456,8 +456,6 @@ parse_macro_expands_to_adt_field = macros cannot expand to {$adt_ty} fields
 
 parse_macro_expands_to_enum_variant = macros cannot expand to enum variants
 
-parse_macro_expands_to_match_arm = macros cannot expand to match arms
-
 parse_macro_invocation_visibility = can't qualify macro invocation with `pub`
     .suggestion = remove the visibility
     .help = try adjusting the macro to put `{$vis}` inside the invocation
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 8921c1c6a03..b11460f2766 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -2823,7 +2823,6 @@ impl<'a> Parser<'a> {
     pub(crate) fn maybe_recover_unexpected_comma(
         &mut self,
         lo: Span,
-        is_mac_invoc: bool,
         rt: CommaRecoveryMode,
     ) -> PResult<'a, ()> {
         if self.token != token::Comma {
@@ -2844,28 +2843,24 @@ impl<'a> Parser<'a> {
         let seq_span = lo.to(self.prev_token.span);
         let mut err = self.struct_span_err(comma_span, "unexpected `,` in pattern");
         if let Ok(seq_snippet) = self.span_to_snippet(seq_span) {
-            if is_mac_invoc {
-                err.note(fluent::parse_macro_expands_to_match_arm);
-            } else {
-                err.multipart_suggestion(
-                    format!(
-                        "try adding parentheses to match on a tuple{}",
-                        if let CommaRecoveryMode::LikelyTuple = rt { "" } else { "..." },
-                    ),
-                    vec![
-                        (seq_span.shrink_to_lo(), "(".to_string()),
-                        (seq_span.shrink_to_hi(), ")".to_string()),
-                    ],
+            err.multipart_suggestion(
+                format!(
+                    "try adding parentheses to match on a tuple{}",
+                    if let CommaRecoveryMode::LikelyTuple = rt { "" } else { "..." },
+                ),
+                vec![
+                    (seq_span.shrink_to_lo(), "(".to_string()),
+                    (seq_span.shrink_to_hi(), ")".to_string()),
+                ],
+                Applicability::MachineApplicable,
+            );
+            if let CommaRecoveryMode::EitherTupleOrPipe = rt {
+                err.span_suggestion(
+                    seq_span,
+                    "...or a vertical bar to match on multiple alternatives",
+                    seq_snippet.replace(',', " |"),
                     Applicability::MachineApplicable,
                 );
-                if let CommaRecoveryMode::EitherTupleOrPipe = rt {
-                    err.span_suggestion(
-                        seq_span,
-                        "...or a vertical bar to match on multiple alternatives",
-                        seq_snippet.replace(',', " |"),
-                        Applicability::MachineApplicable,
-                    );
-                }
             }
         }
         Err(err)
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index 9b8a34cf0cc..3c74a3ea741 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -155,11 +155,7 @@ impl<'a> Parser<'a> {
             Err(err) => return Err(err),
         };
         if rc == RecoverComma::Yes && !first_pat.could_be_never_pattern() {
-            self.maybe_recover_unexpected_comma(
-                first_pat.span,
-                matches!(first_pat.kind, PatKind::MacCall(_)),
-                rt,
-            )?;
+            self.maybe_recover_unexpected_comma(first_pat.span, rt)?;
         }
 
         // If the next token is not a `|`,
@@ -201,7 +197,7 @@ impl<'a> Parser<'a> {
                 err
             })?;
             if rc == RecoverComma::Yes && !pat.could_be_never_pattern() {
-                self.maybe_recover_unexpected_comma(pat.span, false, rt)?;
+                self.maybe_recover_unexpected_comma(pat.span, rt)?;
             }
             pats.push(pat);
         }
diff --git a/tests/ui/parser/macro/macro-expand-to-match-arm.rs b/tests/ui/parser/macro/macro-expand-to-match-arm.rs
index 972ca61cc84..98d2a27884f 100644
--- a/tests/ui/parser/macro/macro-expand-to-match-arm.rs
+++ b/tests/ui/parser/macro/macro-expand-to-match-arm.rs
@@ -3,6 +3,7 @@ macro_rules! arm {
         $pattern => $block
         //~^ ERROR macro expansion ignores token `=>` and any following
         //~| NOTE the usage of `arm!` is likely invalid in pattern context
+        //~| NOTE macros cannot expand to match arms
     };
 }
 
diff --git a/tests/ui/parser/macro/macro-expand-to-match-arm.stderr b/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
index a62109c5050..f162f7dd47b 100644
--- a/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
+++ b/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
@@ -8,6 +8,7 @@ LL |         arm!(None => {}),
    |         ---------------- caused by the macro expansion here
    |
    = note: the usage of `arm!` is likely invalid in pattern context
+   = note: macros cannot expand to match arms
 
 error: aborting due to 1 previous error
 
-- 
cgit 1.4.1-3-g733a5


From a2dcb3a6d9aead3964b3b1cdf814dc7eb9c5d8ed Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 27 Nov 2023 01:53:05 +0100
Subject: Disallow an arm without a body (except for never patterns)

Parsing now accepts a match arm without a body, so we must make sure to
only accept that if the pattern is a never pattern.
---
 compiler/rustc_ast/src/ast.rs                      |  4 +-
 compiler/rustc_ast_lowering/messages.ftl           |  4 ++
 compiler/rustc_ast_lowering/src/errors.rs          |  9 +++
 compiler/rustc_ast_lowering/src/expr.rs            | 13 ++++-
 compiler/rustc_hir/src/hir.rs                      | 17 ++++++
 tests/ui/parser/macro/macro-expand-to-match-arm.rs |  1 +
 .../parser/macro/macro-expand-to-match-arm.stderr  |  8 ++-
 tests/ui/parser/match-arm-without-body.rs          | 18 ++++++
 tests/ui/parser/match-arm-without-body.stderr      | 66 ++++++++++++++++++++--
 9 files changed, 128 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 62bacf97f49..815f5fa8368 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -659,8 +659,8 @@ impl Pat {
         matches!(self.kind, PatKind::Rest)
     }
 
-    /// Could this be a never pattern? I.e. is it a never pattern modulo macro invocations that
-    /// might return never patterns?
+    /// Whether this could be a never pattern, taking into account that a macro invocation can
+    /// return a never pattern. Used to inform errors during parsing.
     pub fn could_be_never_pattern(&self) -> bool {
         let mut could_be_never_pattern = false;
         self.walk(&mut |pat| match &pat.kind {
diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl
index 91591a71611..2a519b418e6 100644
--- a/compiler/rustc_ast_lowering/messages.ftl
+++ b/compiler/rustc_ast_lowering/messages.ftl
@@ -91,6 +91,10 @@ ast_lowering_invalid_register =
 ast_lowering_invalid_register_class =
     invalid register class `{$reg_class}`: {$error}
 
+ast_lowering_match_arm_with_no_body =
+    `match` arm with no body
+    .suggestion = add a body after the pattern
+
 ast_lowering_misplaced_assoc_ty_binding =
     associated type bounds are only allowed in where clauses and function signatures, not in {$position}
 
diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs
index 6e1a9eff500..1bcf4a07eb0 100644
--- a/compiler/rustc_ast_lowering/src/errors.rs
+++ b/compiler/rustc_ast_lowering/src/errors.rs
@@ -340,6 +340,15 @@ pub struct NotSupportedForLifetimeBinderAsyncClosure {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(ast_lowering_match_arm_with_no_body)]
+pub struct MatchArmWithNoBody {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = " => todo!(),", applicability = "has-placeholders")]
+    pub suggestion: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_arbitrary_expression_in_pattern)]
 pub struct ArbitraryExpressionInPattern {
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index b688950f5a3..4a32fa7f929 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1,7 +1,7 @@
 use super::errors::{
     AsyncCoroutinesNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, CoroutineTooManyParameters,
-    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd,
+    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd, MatchArmWithNoBody,
     NotSupportedForLifetimeBinderAsyncClosure, UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
@@ -565,14 +565,21 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
         });
         let hir_id = self.next_id();
+        let span = self.lower_span(arm.span);
         self.lower_attrs(hir_id, &arm.attrs);
         let body = if let Some(body) = &arm.body {
+            // FIXME(never_patterns): Disallow never pattern with a body or guard
             self.lower_expr(body)
         } else {
+            if !pat.is_never_pattern() {
+                self.tcx
+                    .sess
+                    .emit_err(MatchArmWithNoBody { span, suggestion: span.shrink_to_hi() });
+            }
+
             // An arm without a body, meant for never patterns.
             // We add a fake `loop {}` arm body so that it typecks to `!`.
             // FIXME(never_patterns): Desugar into a call to `unreachable_unchecked`.
-            let span = pat.span;
             let block = self.arena.alloc(hir::Block {
                 stmts: &[],
                 expr: None,
@@ -587,7 +594,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 span,
             })
         };
-        hir::Arm { hir_id, pat, guard, body, span: self.lower_span(arm.span) }
+        hir::Arm { hir_id, pat, guard, body, span }
     }
 
     /// Lower an `async` construct to a coroutine that implements `Future`.
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index 81733d8f64e..479a0db75b0 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -1055,6 +1055,23 @@ impl<'hir> Pat<'hir> {
             true
         })
     }
+
+    /// Whether this a never pattern.
+    pub fn is_never_pattern(&self) -> bool {
+        let mut is_never_pattern = false;
+        self.walk(|pat| match &pat.kind {
+            PatKind::Never => {
+                is_never_pattern = true;
+                false
+            }
+            PatKind::Or(s) => {
+                is_never_pattern = s.iter().all(|p| p.is_never_pattern());
+                false
+            }
+            _ => true,
+        });
+        is_never_pattern
+    }
 }
 
 /// A single field in a struct pattern.
diff --git a/tests/ui/parser/macro/macro-expand-to-match-arm.rs b/tests/ui/parser/macro/macro-expand-to-match-arm.rs
index 98d2a27884f..db38fa0d7bc 100644
--- a/tests/ui/parser/macro/macro-expand-to-match-arm.rs
+++ b/tests/ui/parser/macro/macro-expand-to-match-arm.rs
@@ -13,6 +13,7 @@ fn main() {
         Some(1) => {},
         arm!(None => {}),
         //~^ NOTE caused by the macro expansion here
+        //~| ERROR `match` arm with no body
         Some(2) => {},
         _ => {},
     };
diff --git a/tests/ui/parser/macro/macro-expand-to-match-arm.stderr b/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
index f162f7dd47b..e3e7ff89c81 100644
--- a/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
+++ b/tests/ui/parser/macro/macro-expand-to-match-arm.stderr
@@ -10,5 +10,11 @@ LL |         arm!(None => {}),
    = note: the usage of `arm!` is likely invalid in pattern context
    = note: macros cannot expand to match arms
 
-error: aborting due to 1 previous error
+error: `match` arm with no body
+  --> $DIR/macro-expand-to-match-arm.rs:14:9
+   |
+LL |         arm!(None => {}),
+   |         ^^^^^^^^^^^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: aborting due to 2 previous errors
 
diff --git a/tests/ui/parser/match-arm-without-body.rs b/tests/ui/parser/match-arm-without-body.rs
index c33bd0c3031..c3487c2c658 100644
--- a/tests/ui/parser/match-arm-without-body.rs
+++ b/tests/ui/parser/match-arm-without-body.rs
@@ -5,6 +5,8 @@ macro_rules! pat {
 fn main() {
     match Some(false) {
         Some(_)
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
     }
     match Some(false) {
         Some(_)
@@ -26,6 +28,8 @@ fn main() {
     }
     match Some(false) {
         Some(_) if true
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
     }
     match Some(false) {
         Some(_) if true
@@ -34,28 +38,42 @@ fn main() {
     }
     match Some(false) {
         Some(_) if true,
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
     }
     match Some(false) {
         Some(_) if true,
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
         _ => {}
     }
     match Some(false) {
         pat!()
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
     }
     match Some(false) {
         pat!(),
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
     }
     match Some(false) {
         pat!() if true,
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
     }
     match Some(false) {
         pat!()
         //~^ ERROR expected `,` following `match` arm
         //~| HELP missing a comma here
+        //~| ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
         _ => {}
     }
     match Some(false) {
         pat!(),
+        //~^ ERROR `match` arm with no body
+        //~| HELP add a body after the pattern
         _ => {}
     }
 }
diff --git a/tests/ui/parser/match-arm-without-body.stderr b/tests/ui/parser/match-arm-without-body.stderr
index 77fd9176633..3a06ed050b5 100644
--- a/tests/ui/parser/match-arm-without-body.stderr
+++ b/tests/ui/parser/match-arm-without-body.stderr
@@ -1,5 +1,5 @@
 error: expected one of `,`, `=>`, `if`, `|`, or `}`, found reserved identifier `_`
-  --> $DIR/match-arm-without-body.rs:11:9
+  --> $DIR/match-arm-without-body.rs:13:9
    |
 LL |         Some(_)
    |                - expected one of `,`, `=>`, `if`, `|`, or `}`
@@ -7,7 +7,7 @@ LL |         _ => {}
    |         ^ unexpected token
 
 error: unexpected `,` in pattern
-  --> $DIR/match-arm-without-body.rs:15:16
+  --> $DIR/match-arm-without-body.rs:17:16
    |
 LL |         Some(_),
    |                ^
@@ -22,7 +22,7 @@ LL |         Some(_) |
    |
 
 error: unexpected `,` in pattern
-  --> $DIR/match-arm-without-body.rs:21:16
+  --> $DIR/match-arm-without-body.rs:23:16
    |
 LL |         Some(_),
    |                ^
@@ -45,7 +45,7 @@ LL ~         _ => {}
    |
 
 error: expected one of `,`, `.`, `=>`, `?`, `}`, or an operator, found reserved identifier `_`
-  --> $DIR/match-arm-without-body.rs:32:9
+  --> $DIR/match-arm-without-body.rs:36:9
    |
 LL |         Some(_) if true
    |                        - expected one of `,`, `.`, `=>`, `?`, `}`, or an operator
@@ -53,10 +53,64 @@ LL |         _ => {}
    |         ^ unexpected token
 
 error: expected `,` following `match` arm
-  --> $DIR/match-arm-without-body.rs:52:15
+  --> $DIR/match-arm-without-body.rs:66:15
    |
 LL |         pat!()
    |               ^ help: missing a comma here to end this `match` arm: `,`
 
-error: aborting due to 5 previous errors
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:7:9
+   |
+LL |         Some(_)
+   |         ^^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:30:9
+   |
+LL |         Some(_) if true
+   |         ^^^^^^^^^^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:40:9
+   |
+LL |         Some(_) if true,
+   |         ^^^^^^^^^^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:45:9
+   |
+LL |         Some(_) if true,
+   |         ^^^^^^^^^^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:51:9
+   |
+LL |         pat!()
+   |         ^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:56:9
+   |
+LL |         pat!(),
+   |         ^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:61:9
+   |
+LL |         pat!() if true,
+   |         ^^^^^^^^^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:66:9
+   |
+LL |         pat!()
+   |         ^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: `match` arm with no body
+  --> $DIR/match-arm-without-body.rs:74:9
+   |
+LL |         pat!(),
+   |         ^^^^^^- help: add a body after the pattern: `=> todo!(),`
+
+error: aborting due to 14 previous errors
 
-- 
cgit 1.4.1-3-g733a5


From 06a8ed10b627d04c9e91c41064c38745374acc71 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 27 Nov 2023 03:47:49 +0100
Subject: Disallow guards on never patterns

---
 compiler/rustc_ast_lowering/messages.ftl  |  4 +++
 compiler/rustc_ast_lowering/src/errors.rs |  8 ++++++
 compiler/rustc_ast_lowering/src/expr.rs   |  7 ++++--
 tests/ui/never_patterns/check.rs          |  3 +--
 tests/ui/never_patterns/check.stderr      | 41 ++++---------------------------
 tests/ui/never_patterns/parse.rs          |  3 +++
 tests/ui/never_patterns/parse.stderr      | 24 +++++++++++++++---
 7 files changed, 47 insertions(+), 43 deletions(-)

diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl
index 2a519b418e6..ecbe8cc6aec 100644
--- a/compiler/rustc_ast_lowering/messages.ftl
+++ b/compiler/rustc_ast_lowering/messages.ftl
@@ -108,6 +108,10 @@ ast_lowering_misplaced_impl_trait =
 ast_lowering_misplaced_relax_trait_bound =
     `?Trait` bounds are only permitted at the point where a type parameter is declared
 
+ast_lowering_never_pattern_with_guard =
+    a guard on a never pattern will never be run
+    .suggestion = remove this guard
+
 ast_lowering_not_supported_for_lifetime_binder_async_closure =
     `for<...>` binders on `async` closures are not currently supported
 
diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs
index 1bcf4a07eb0..c6a4166f537 100644
--- a/compiler/rustc_ast_lowering/src/errors.rs
+++ b/compiler/rustc_ast_lowering/src/errors.rs
@@ -349,6 +349,14 @@ pub struct MatchArmWithNoBody {
     pub suggestion: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(ast_lowering_never_pattern_with_guard)]
+pub struct NeverPatternWithGuard {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_arbitrary_expression_in_pattern)]
 pub struct ArbitraryExpressionInPattern {
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 4a32fa7f929..4913f9cd12d 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -2,7 +2,7 @@ use super::errors::{
     AsyncCoroutinesNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, CoroutineTooManyParameters,
     FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd, MatchArmWithNoBody,
-    NotSupportedForLifetimeBinderAsyncClosure, UnderscoreExprLhsAssign,
+    NeverPatternWithGuard, NotSupportedForLifetimeBinderAsyncClosure, UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
 use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs};
@@ -550,7 +550,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
     fn lower_arm(&mut self, arm: &Arm) -> hir::Arm<'hir> {
         let pat = self.lower_pat(&arm.pat);
-        let guard = arm.guard.as_ref().map(|cond| {
+        let mut guard = arm.guard.as_ref().map(|cond| {
             if let ExprKind::Let(pat, scrutinee, span, is_recovered) = &cond.kind {
                 hir::Guard::IfLet(self.arena.alloc(hir::Let {
                     hir_id: self.next_id(),
@@ -575,6 +575,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 self.tcx
                     .sess
                     .emit_err(MatchArmWithNoBody { span, suggestion: span.shrink_to_hi() });
+            } else if let Some(g) = &arm.guard {
+                self.tcx.sess.emit_err(NeverPatternWithGuard { span: g.span });
+                guard = None;
             }
 
             // An arm without a body, meant for never patterns.
diff --git a/tests/ui/never_patterns/check.rs b/tests/ui/never_patterns/check.rs
index 9b02fc7996d..1f55ef11242 100644
--- a/tests/ui/never_patterns/check.rs
+++ b/tests/ui/never_patterns/check.rs
@@ -15,12 +15,11 @@ fn no_arms_or_guards(x: Void) {
         None => {}
     }
     match None::<Void> {
-        //~^ ERROR non-exhaustive
         Some(!) if true,
+        //~^ ERROR guard on a never pattern
         None => {}
     }
     match None::<Void> {
-        //~^ ERROR non-exhaustive
         Some(!) if true => {}
         None => {}
     }
diff --git a/tests/ui/never_patterns/check.stderr b/tests/ui/never_patterns/check.stderr
index 945812225c4..a25ee1b0a2a 100644
--- a/tests/ui/never_patterns/check.stderr
+++ b/tests/ui/never_patterns/check.stderr
@@ -1,39 +1,8 @@
-error[E0004]: non-exhaustive patterns: `Some(_)` not covered
-  --> $DIR/check.rs:17:11
-   |
-LL |     match None::<Void> {
-   |           ^^^^^^^^^^^^ pattern `Some(_)` not covered
-   |
-note: `Option<Void>` defined here
-  --> $SRC_DIR/core/src/option.rs:LL:COL
-  ::: $SRC_DIR/core/src/option.rs:LL:COL
-   |
-   = note: not covered
-   = note: the matched value is of type `Option<Void>`
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
-   |
-LL ~         None => {},
-LL +         Some(_) => todo!()
-   |
-
-error[E0004]: non-exhaustive patterns: `Some(_)` not covered
-  --> $DIR/check.rs:22:11
-   |
-LL |     match None::<Void> {
-   |           ^^^^^^^^^^^^ pattern `Some(_)` not covered
-   |
-note: `Option<Void>` defined here
-  --> $SRC_DIR/core/src/option.rs:LL:COL
-  ::: $SRC_DIR/core/src/option.rs:LL:COL
-   |
-   = note: not covered
-   = note: the matched value is of type `Option<Void>`
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
-   |
-LL ~         None => {},
-LL +         Some(_) => todo!()
+error: a guard on a never pattern will never be run
+  --> $DIR/check.rs:18:20
    |
+LL |         Some(!) if true,
+   |                    ^^^^ help: remove this guard
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
-For more information about this error, try `rustc --explain E0004`.
diff --git a/tests/ui/never_patterns/parse.rs b/tests/ui/never_patterns/parse.rs
index 850416d723a..1b23e60e0ca 100644
--- a/tests/ui/never_patterns/parse.rs
+++ b/tests/ui/never_patterns/parse.rs
@@ -30,10 +30,12 @@ fn parse(x: Void) {
     match None::<Void> {
         Some(!) if true
         //~^ ERROR expected `,` following `match` arm
+        //~| ERROR guard on a never pattern
         None => {}
     }
     match None::<Void> {
         Some(!) if true,
+        //~^ ERROR guard on a never pattern
         None => {}
     }
     match None::<Void> {
@@ -45,6 +47,7 @@ fn parse(x: Void) {
     }
     match x {
         never!() if true,
+        //~^ ERROR guard on a never pattern
     }
     match x {
         never!()
diff --git a/tests/ui/never_patterns/parse.stderr b/tests/ui/never_patterns/parse.stderr
index 7ea33540c5e..e81a13a3967 100644
--- a/tests/ui/never_patterns/parse.stderr
+++ b/tests/ui/never_patterns/parse.stderr
@@ -11,16 +11,34 @@ LL |         Some(!) if true
    |                        ^ help: missing a comma here to end this `match` arm: `,`
 
 error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `<=`
-  --> $DIR/parse.rs:40:17
+  --> $DIR/parse.rs:42:17
    |
 LL |         Some(!) <=
    |                 ^^ expected one of `,`, `=>`, `if`, `|`, or `}`
 
 error: top-level or-patterns are not allowed in `let` bindings
-  --> $DIR/parse.rs:64:9
+  --> $DIR/parse.rs:67:9
    |
 LL |     let Ok(_) | Err(!) = &res; // Disallowed; see #82048.
    |         ^^^^^^^^^^^^^^ help: wrap the pattern in parentheses: `(Ok(_) | Err(!))`
 
-error: aborting due to 4 previous errors
+error: a guard on a never pattern will never be run
+  --> $DIR/parse.rs:31:20
+   |
+LL |         Some(!) if true
+   |                    ^^^^ help: remove this guard
+
+error: a guard on a never pattern will never be run
+  --> $DIR/parse.rs:37:20
+   |
+LL |         Some(!) if true,
+   |                    ^^^^ help: remove this guard
+
+error: a guard on a never pattern will never be run
+  --> $DIR/parse.rs:49:21
+   |
+LL |         never!() if true,
+   |                     ^^^^ help: remove this guard
+
+error: aborting due to 7 previous errors
 
-- 
cgit 1.4.1-3-g733a5


From 70deb9a57f4c5757162f4c845ee9fe318717064f Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 27 Nov 2023 04:08:09 +0100
Subject: Disallow arm bodies on never patterns

---
 compiler/rustc_ast_lowering/messages.ftl           |  5 ++++
 compiler/rustc_ast_lowering/src/errors.rs          |  9 +++++++
 compiler/rustc_ast_lowering/src/expr.rs            | 21 +++++++++------
 .../feature-gates/feature-gate-never_patterns.rs   |  2 +-
 .../feature-gate-never_patterns.stderr             |  2 +-
 tests/ui/never_patterns/check.rs                   |  3 +++
 tests/ui/never_patterns/check.stderr               | 31 ++++++++++++++++++++--
 tests/ui/pattern/never_patterns.rs                 | 28 +++++++++----------
 8 files changed, 75 insertions(+), 26 deletions(-)

diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl
index ecbe8cc6aec..6bde4f2d8fa 100644
--- a/compiler/rustc_ast_lowering/messages.ftl
+++ b/compiler/rustc_ast_lowering/messages.ftl
@@ -108,6 +108,11 @@ ast_lowering_misplaced_impl_trait =
 ast_lowering_misplaced_relax_trait_bound =
     `?Trait` bounds are only permitted at the point where a type parameter is declared
 
+ast_lowering_never_pattern_with_body =
+    a never pattern is always unreachable
+    .label = this will never be executed
+    .suggestion = remove this expression
+
 ast_lowering_never_pattern_with_guard =
     a guard on a never pattern will never be run
     .suggestion = remove this guard
diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs
index c6a4166f537..11bb559719b 100644
--- a/compiler/rustc_ast_lowering/src/errors.rs
+++ b/compiler/rustc_ast_lowering/src/errors.rs
@@ -349,6 +349,15 @@ pub struct MatchArmWithNoBody {
     pub suggestion: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(ast_lowering_never_pattern_with_body)]
+pub struct NeverPatternWithBody {
+    #[primary_span]
+    #[label]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(ast_lowering_never_pattern_with_guard)]
 pub struct NeverPatternWithGuard {
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 4913f9cd12d..137ccb99346 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -2,7 +2,8 @@ use super::errors::{
     AsyncCoroutinesNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, CoroutineTooManyParameters,
     FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd, MatchArmWithNoBody,
-    NeverPatternWithGuard, NotSupportedForLifetimeBinderAsyncClosure, UnderscoreExprLhsAssign,
+    NeverPatternWithBody, NeverPatternWithGuard, NotSupportedForLifetimeBinderAsyncClosure,
+    UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
 use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs};
@@ -567,20 +568,24 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let hir_id = self.next_id();
         let span = self.lower_span(arm.span);
         self.lower_attrs(hir_id, &arm.attrs);
-        let body = if let Some(body) = &arm.body {
-            // FIXME(never_patterns): Disallow never pattern with a body or guard
+        let is_never_pattern = pat.is_never_pattern();
+        let body = if let Some(body) = &arm.body
+            && !is_never_pattern
+        {
             self.lower_expr(body)
         } else {
-            if !pat.is_never_pattern() {
-                self.tcx
-                    .sess
-                    .emit_err(MatchArmWithNoBody { span, suggestion: span.shrink_to_hi() });
+            // Either `body.is_none()` or `is_never_pattern` here.
+            if !is_never_pattern {
+                let suggestion = span.shrink_to_hi();
+                self.tcx.sess.emit_err(MatchArmWithNoBody { span, suggestion });
+            } else if let Some(body) = &arm.body {
+                self.tcx.sess.emit_err(NeverPatternWithBody { span: body.span });
+                guard = None;
             } else if let Some(g) = &arm.guard {
                 self.tcx.sess.emit_err(NeverPatternWithGuard { span: g.span });
                 guard = None;
             }
 
-            // An arm without a body, meant for never patterns.
             // We add a fake `loop {}` arm body so that it typecks to `!`.
             // FIXME(never_patterns): Desugar into a call to `unreachable_unchecked`.
             let block = self.arena.alloc(hir::Block {
diff --git a/tests/ui/feature-gates/feature-gate-never_patterns.rs b/tests/ui/feature-gates/feature-gate-never_patterns.rs
index 69e9f62abf0..ca5ce3b9489 100644
--- a/tests/ui/feature-gates/feature-gate-never_patterns.rs
+++ b/tests/ui/feature-gates/feature-gate-never_patterns.rs
@@ -12,7 +12,7 @@ fn main() {
     unsafe {
         let ptr: *const Void = NonNull::dangling().as_ptr();
         match *ptr {
-            ! => {} //~ ERROR `!` patterns are experimental
+            ! //~ ERROR `!` patterns are experimental
         }
     }
 
diff --git a/tests/ui/feature-gates/feature-gate-never_patterns.stderr b/tests/ui/feature-gates/feature-gate-never_patterns.stderr
index b7290eeb36d..2354a3b0476 100644
--- a/tests/ui/feature-gates/feature-gate-never_patterns.stderr
+++ b/tests/ui/feature-gates/feature-gate-never_patterns.stderr
@@ -18,7 +18,7 @@ LL |     let (Ok(_x) | Err(&!)) = res.as_ref();
 error[E0658]: `!` patterns are experimental
   --> $DIR/feature-gate-never_patterns.rs:15:13
    |
-LL |             ! => {}
+LL |             !
    |             ^
    |
    = note: see issue #118155 <https://github.com/rust-lang/rust/issues/118155> for more information
diff --git a/tests/ui/never_patterns/check.rs b/tests/ui/never_patterns/check.rs
index 1f55ef11242..e298112244a 100644
--- a/tests/ui/never_patterns/check.rs
+++ b/tests/ui/never_patterns/check.rs
@@ -12,6 +12,7 @@ macro_rules! never {
 fn no_arms_or_guards(x: Void) {
     match None::<Void> {
         Some(!) => {}
+        //~^ ERROR a never pattern is always unreachable
         None => {}
     }
     match None::<Void> {
@@ -21,10 +22,12 @@ fn no_arms_or_guards(x: Void) {
     }
     match None::<Void> {
         Some(!) if true => {}
+        //~^ ERROR a never pattern is always unreachable
         None => {}
     }
     match None::<Void> {
         Some(never!()) => {},
+        //~^ ERROR a never pattern is always unreachable
         None => {}
     }
 }
diff --git a/tests/ui/never_patterns/check.stderr b/tests/ui/never_patterns/check.stderr
index a25ee1b0a2a..bfbc7a1b534 100644
--- a/tests/ui/never_patterns/check.stderr
+++ b/tests/ui/never_patterns/check.stderr
@@ -1,8 +1,35 @@
+error: a never pattern is always unreachable
+  --> $DIR/check.rs:14:20
+   |
+LL |         Some(!) => {}
+   |                    ^^
+   |                    |
+   |                    this will never be executed
+   |                    help: remove this expression
+
 error: a guard on a never pattern will never be run
-  --> $DIR/check.rs:18:20
+  --> $DIR/check.rs:19:20
    |
 LL |         Some(!) if true,
    |                    ^^^^ help: remove this guard
 
-error: aborting due to 1 previous error
+error: a never pattern is always unreachable
+  --> $DIR/check.rs:24:28
+   |
+LL |         Some(!) if true => {}
+   |                            ^^
+   |                            |
+   |                            this will never be executed
+   |                            help: remove this expression
+
+error: a never pattern is always unreachable
+  --> $DIR/check.rs:29:27
+   |
+LL |         Some(never!()) => {},
+   |                           ^^
+   |                           |
+   |                           this will never be executed
+   |                           help: remove this expression
+
+error: aborting due to 4 previous errors
 
diff --git a/tests/ui/pattern/never_patterns.rs b/tests/ui/pattern/never_patterns.rs
index fdba1b8e087..915f3e75e7b 100644
--- a/tests/ui/pattern/never_patterns.rs
+++ b/tests/ui/pattern/never_patterns.rs
@@ -20,58 +20,58 @@ fn never_pattern_location(void: Void) {
     // FIXME(never_patterns): Don't accept on a non-empty type.
     match Some(0) {
         None => {}
-        Some(!) => {}
+        Some(!),
     }
     // FIXME(never_patterns): Don't accept on an arbitrary type, even if there are no more branches.
     match () {
         () => {}
-        ! => {}
+        !,
     }
     // FIXME(never_patterns): Don't accept even on an empty branch.
     match None::<Void> {
         None => {}
-        ! => {}
+        !,
     }
     // FIXME(never_patterns): Let alone if the emptiness is behind a reference.
     match None::<&Void> {
         None => {}
-        ! => {}
+        !,
     }
     // Participate in match ergonomics.
     match &void {
-        ! => {}
+        !
     }
     match &&void {
-        ! => {}
+        !
     }
     match &&void {
-        &! => {}
+        &!
     }
     match &None::<Void> {
         None => {}
-        Some(!) => {}
+        Some(!)
     }
     match None::<&Void> {
         None => {}
-        Some(!) => {}
+        Some(!),
     }
     // Accept on a composite empty type.
     match None::<&(u32, Void)> {
         None => {}
-        Some(&!) => {}
+        Some(&!),
     }
     // Accept on an simple empty type.
     match None::<Void> {
         None => {}
-        Some(!) => {}
+        Some(!),
     }
     match None::<&Void> {
         None => {}
-        Some(&!) => {}
+        Some(&!),
     }
     match None::<&(u32, Void)> {
         None => {}
-        Some(&(_, !)) => {}
+        Some(&(_, !)),
     }
 }
 
@@ -89,7 +89,7 @@ fn never_and_bindings() {
     // FIXME(never_patterns): A never pattern mustn't have bindings.
     match x {
         Ok(_) => {}
-        Err(&(_b, !)) => {}
+        Err(&(_b, !)),
     }
     match x {
         Ok(_a) | Err(&(_b, !)) => {}
-- 
cgit 1.4.1-3-g733a5


From 431cc4a3b0ea6e0ef2244d2e6ff2ad89ccd23c74 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 2 Dec 2023 05:41:08 +0100
Subject: Satisfy tidy

---
 tests/ui/never_patterns/check.rs                   | 33 ----------
 tests/ui/never_patterns/check.stderr               | 35 -----------
 tests/ui/never_patterns/parse.rs                   | 71 ----------------------
 tests/ui/never_patterns/parse.stderr               | 44 --------------
 tests/ui/rfcs/rfc-0000-never_patterns/check.rs     | 33 ++++++++++
 tests/ui/rfcs/rfc-0000-never_patterns/check.stderr | 35 +++++++++++
 tests/ui/rfcs/rfc-0000-never_patterns/parse.rs     | 71 ++++++++++++++++++++++
 tests/ui/rfcs/rfc-0000-never_patterns/parse.stderr | 44 ++++++++++++++
 8 files changed, 183 insertions(+), 183 deletions(-)
 delete mode 100644 tests/ui/never_patterns/check.rs
 delete mode 100644 tests/ui/never_patterns/check.stderr
 delete mode 100644 tests/ui/never_patterns/parse.rs
 delete mode 100644 tests/ui/never_patterns/parse.stderr
 create mode 100644 tests/ui/rfcs/rfc-0000-never_patterns/check.rs
 create mode 100644 tests/ui/rfcs/rfc-0000-never_patterns/check.stderr
 create mode 100644 tests/ui/rfcs/rfc-0000-never_patterns/parse.rs
 create mode 100644 tests/ui/rfcs/rfc-0000-never_patterns/parse.stderr

diff --git a/tests/ui/never_patterns/check.rs b/tests/ui/never_patterns/check.rs
deleted file mode 100644
index e298112244a..00000000000
--- a/tests/ui/never_patterns/check.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-#![feature(never_patterns)]
-#![allow(incomplete_features)]
-
-enum Void {}
-
-fn main() {}
-
-macro_rules! never {
-    () => { ! }
-}
-
-fn no_arms_or_guards(x: Void) {
-    match None::<Void> {
-        Some(!) => {}
-        //~^ ERROR a never pattern is always unreachable
-        None => {}
-    }
-    match None::<Void> {
-        Some(!) if true,
-        //~^ ERROR guard on a never pattern
-        None => {}
-    }
-    match None::<Void> {
-        Some(!) if true => {}
-        //~^ ERROR a never pattern is always unreachable
-        None => {}
-    }
-    match None::<Void> {
-        Some(never!()) => {},
-        //~^ ERROR a never pattern is always unreachable
-        None => {}
-    }
-}
diff --git a/tests/ui/never_patterns/check.stderr b/tests/ui/never_patterns/check.stderr
deleted file mode 100644
index bfbc7a1b534..00000000000
--- a/tests/ui/never_patterns/check.stderr
+++ /dev/null
@@ -1,35 +0,0 @@
-error: a never pattern is always unreachable
-  --> $DIR/check.rs:14:20
-   |
-LL |         Some(!) => {}
-   |                    ^^
-   |                    |
-   |                    this will never be executed
-   |                    help: remove this expression
-
-error: a guard on a never pattern will never be run
-  --> $DIR/check.rs:19:20
-   |
-LL |         Some(!) if true,
-   |                    ^^^^ help: remove this guard
-
-error: a never pattern is always unreachable
-  --> $DIR/check.rs:24:28
-   |
-LL |         Some(!) if true => {}
-   |                            ^^
-   |                            |
-   |                            this will never be executed
-   |                            help: remove this expression
-
-error: a never pattern is always unreachable
-  --> $DIR/check.rs:29:27
-   |
-LL |         Some(never!()) => {},
-   |                           ^^
-   |                           |
-   |                           this will never be executed
-   |                           help: remove this expression
-
-error: aborting due to 4 previous errors
-
diff --git a/tests/ui/never_patterns/parse.rs b/tests/ui/never_patterns/parse.rs
deleted file mode 100644
index 1b23e60e0ca..00000000000
--- a/tests/ui/never_patterns/parse.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-#![feature(never_patterns)]
-#![allow(incomplete_features)]
-
-enum Void {}
-
-fn main() {}
-
-macro_rules! never {
-    () => { ! }
-}
-
-fn parse(x: Void) {
-    match None::<Void> {
-        None => {}
-        Some(!),
-    }
-    match None::<Void> {
-        Some(!),
-        None => {}
-    }
-    match None::<Void> {
-        None => {}
-        Some(!)
-    }
-    match None::<Void> {
-        Some(!)
-        //~^ ERROR expected `,` following `match` arm
-        None => {}
-    }
-    match None::<Void> {
-        Some(!) if true
-        //~^ ERROR expected `,` following `match` arm
-        //~| ERROR guard on a never pattern
-        None => {}
-    }
-    match None::<Void> {
-        Some(!) if true,
-        //~^ ERROR guard on a never pattern
-        None => {}
-    }
-    match None::<Void> {
-        Some(!) <=
-        //~^ ERROR expected one of
-    }
-    match x {
-        never!(),
-    }
-    match x {
-        never!() if true,
-        //~^ ERROR guard on a never pattern
-    }
-    match x {
-        never!()
-    }
-    match &x {
-        &never!(),
-    }
-    match None::<Void> {
-        Some(never!()),
-        None => {}
-    }
-    match x { ! }
-    match &x { &! }
-
-    let res: Result<bool, Void> = Ok(false);
-    let Ok(_) = res;
-    let Ok(_) | Err(!) = &res; // Disallowed; see #82048.
-    //~^ ERROR top-level or-patterns are not allowed in `let` bindings
-    let (Ok(_) | Err(!)) = &res;
-    let (Ok(_) | Err(&!)) = res.as_ref();
-}
diff --git a/tests/ui/never_patterns/parse.stderr b/tests/ui/never_patterns/parse.stderr
deleted file mode 100644
index e81a13a3967..00000000000
--- a/tests/ui/never_patterns/parse.stderr
+++ /dev/null
@@ -1,44 +0,0 @@
-error: expected `,` following `match` arm
-  --> $DIR/parse.rs:26:16
-   |
-LL |         Some(!)
-   |                ^ help: missing a comma here to end this `match` arm: `,`
-
-error: expected `,` following `match` arm
-  --> $DIR/parse.rs:31:24
-   |
-LL |         Some(!) if true
-   |                        ^ help: missing a comma here to end this `match` arm: `,`
-
-error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `<=`
-  --> $DIR/parse.rs:42:17
-   |
-LL |         Some(!) <=
-   |                 ^^ expected one of `,`, `=>`, `if`, `|`, or `}`
-
-error: top-level or-patterns are not allowed in `let` bindings
-  --> $DIR/parse.rs:67:9
-   |
-LL |     let Ok(_) | Err(!) = &res; // Disallowed; see #82048.
-   |         ^^^^^^^^^^^^^^ help: wrap the pattern in parentheses: `(Ok(_) | Err(!))`
-
-error: a guard on a never pattern will never be run
-  --> $DIR/parse.rs:31:20
-   |
-LL |         Some(!) if true
-   |                    ^^^^ help: remove this guard
-
-error: a guard on a never pattern will never be run
-  --> $DIR/parse.rs:37:20
-   |
-LL |         Some(!) if true,
-   |                    ^^^^ help: remove this guard
-
-error: a guard on a never pattern will never be run
-  --> $DIR/parse.rs:49:21
-   |
-LL |         never!() if true,
-   |                     ^^^^ help: remove this guard
-
-error: aborting due to 7 previous errors
-
diff --git a/tests/ui/rfcs/rfc-0000-never_patterns/check.rs b/tests/ui/rfcs/rfc-0000-never_patterns/check.rs
new file mode 100644
index 00000000000..e298112244a
--- /dev/null
+++ b/tests/ui/rfcs/rfc-0000-never_patterns/check.rs
@@ -0,0 +1,33 @@
+#![feature(never_patterns)]
+#![allow(incomplete_features)]
+
+enum Void {}
+
+fn main() {}
+
+macro_rules! never {
+    () => { ! }
+}
+
+fn no_arms_or_guards(x: Void) {
+    match None::<Void> {
+        Some(!) => {}
+        //~^ ERROR a never pattern is always unreachable
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true,
+        //~^ ERROR guard on a never pattern
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true => {}
+        //~^ ERROR a never pattern is always unreachable
+        None => {}
+    }
+    match None::<Void> {
+        Some(never!()) => {},
+        //~^ ERROR a never pattern is always unreachable
+        None => {}
+    }
+}
diff --git a/tests/ui/rfcs/rfc-0000-never_patterns/check.stderr b/tests/ui/rfcs/rfc-0000-never_patterns/check.stderr
new file mode 100644
index 00000000000..bfbc7a1b534
--- /dev/null
+++ b/tests/ui/rfcs/rfc-0000-never_patterns/check.stderr
@@ -0,0 +1,35 @@
+error: a never pattern is always unreachable
+  --> $DIR/check.rs:14:20
+   |
+LL |         Some(!) => {}
+   |                    ^^
+   |                    |
+   |                    this will never be executed
+   |                    help: remove this expression
+
+error: a guard on a never pattern will never be run
+  --> $DIR/check.rs:19:20
+   |
+LL |         Some(!) if true,
+   |                    ^^^^ help: remove this guard
+
+error: a never pattern is always unreachable
+  --> $DIR/check.rs:24:28
+   |
+LL |         Some(!) if true => {}
+   |                            ^^
+   |                            |
+   |                            this will never be executed
+   |                            help: remove this expression
+
+error: a never pattern is always unreachable
+  --> $DIR/check.rs:29:27
+   |
+LL |         Some(never!()) => {},
+   |                           ^^
+   |                           |
+   |                           this will never be executed
+   |                           help: remove this expression
+
+error: aborting due to 4 previous errors
+
diff --git a/tests/ui/rfcs/rfc-0000-never_patterns/parse.rs b/tests/ui/rfcs/rfc-0000-never_patterns/parse.rs
new file mode 100644
index 00000000000..1b23e60e0ca
--- /dev/null
+++ b/tests/ui/rfcs/rfc-0000-never_patterns/parse.rs
@@ -0,0 +1,71 @@
+#![feature(never_patterns)]
+#![allow(incomplete_features)]
+
+enum Void {}
+
+fn main() {}
+
+macro_rules! never {
+    () => { ! }
+}
+
+fn parse(x: Void) {
+    match None::<Void> {
+        None => {}
+        Some(!),
+    }
+    match None::<Void> {
+        Some(!),
+        None => {}
+    }
+    match None::<Void> {
+        None => {}
+        Some(!)
+    }
+    match None::<Void> {
+        Some(!)
+        //~^ ERROR expected `,` following `match` arm
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true
+        //~^ ERROR expected `,` following `match` arm
+        //~| ERROR guard on a never pattern
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) if true,
+        //~^ ERROR guard on a never pattern
+        None => {}
+    }
+    match None::<Void> {
+        Some(!) <=
+        //~^ ERROR expected one of
+    }
+    match x {
+        never!(),
+    }
+    match x {
+        never!() if true,
+        //~^ ERROR guard on a never pattern
+    }
+    match x {
+        never!()
+    }
+    match &x {
+        &never!(),
+    }
+    match None::<Void> {
+        Some(never!()),
+        None => {}
+    }
+    match x { ! }
+    match &x { &! }
+
+    let res: Result<bool, Void> = Ok(false);
+    let Ok(_) = res;
+    let Ok(_) | Err(!) = &res; // Disallowed; see #82048.
+    //~^ ERROR top-level or-patterns are not allowed in `let` bindings
+    let (Ok(_) | Err(!)) = &res;
+    let (Ok(_) | Err(&!)) = res.as_ref();
+}
diff --git a/tests/ui/rfcs/rfc-0000-never_patterns/parse.stderr b/tests/ui/rfcs/rfc-0000-never_patterns/parse.stderr
new file mode 100644
index 00000000000..e81a13a3967
--- /dev/null
+++ b/tests/ui/rfcs/rfc-0000-never_patterns/parse.stderr
@@ -0,0 +1,44 @@
+error: expected `,` following `match` arm
+  --> $DIR/parse.rs:26:16
+   |
+LL |         Some(!)
+   |                ^ help: missing a comma here to end this `match` arm: `,`
+
+error: expected `,` following `match` arm
+  --> $DIR/parse.rs:31:24
+   |
+LL |         Some(!) if true
+   |                        ^ help: missing a comma here to end this `match` arm: `,`
+
+error: expected one of `,`, `=>`, `if`, `|`, or `}`, found `<=`
+  --> $DIR/parse.rs:42:17
+   |
+LL |         Some(!) <=
+   |                 ^^ expected one of `,`, `=>`, `if`, `|`, or `}`
+
+error: top-level or-patterns are not allowed in `let` bindings
+  --> $DIR/parse.rs:67:9
+   |
+LL |     let Ok(_) | Err(!) = &res; // Disallowed; see #82048.
+   |         ^^^^^^^^^^^^^^ help: wrap the pattern in parentheses: `(Ok(_) | Err(!))`
+
+error: a guard on a never pattern will never be run
+  --> $DIR/parse.rs:31:20
+   |
+LL |         Some(!) if true
+   |                    ^^^^ help: remove this guard
+
+error: a guard on a never pattern will never be run
+  --> $DIR/parse.rs:37:20
+   |
+LL |         Some(!) if true,
+   |                    ^^^^ help: remove this guard
+
+error: a guard on a never pattern will never be run
+  --> $DIR/parse.rs:49:21
+   |
+LL |         never!() if true,
+   |                     ^^^^ help: remove this guard
+
+error: aborting due to 7 previous errors
+
-- 
cgit 1.4.1-3-g733a5


From 199098b71b7495fbbc91d62e20b831f17761c95b Mon Sep 17 00:00:00 2001
From: bohan <bohan-zhang@foxmail.com>
Date: Tue, 21 Nov 2023 02:42:09 +0800
Subject: dedup for duplicate suggestions

---
 compiler/rustc_errors/src/diagnostic.rs |  9 +++++----
 tests/ui/macros/issue-118048.rs         | 10 ++++++++++
 tests/ui/macros/issue-118048.stderr     | 21 +++++++++++++++++++++
 3 files changed, 36 insertions(+), 4 deletions(-)
 create mode 100644 tests/ui/macros/issue-118048.rs
 create mode 100644 tests/ui/macros/issue-118048.stderr

diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs
index 0aaa8ae69e1..d222010901f 100644
--- a/compiler/rustc_errors/src/diagnostic.rs
+++ b/compiler/rustc_errors/src/diagnostic.rs
@@ -622,17 +622,18 @@ impl Diagnostic {
     pub fn multipart_suggestion_with_style(
         &mut self,
         msg: impl Into<SubdiagnosticMessage>,
-        suggestion: Vec<(Span, String)>,
+        mut suggestion: Vec<(Span, String)>,
         applicability: Applicability,
         style: SuggestionStyle,
     ) -> &mut Self {
-        let mut parts = suggestion
+        suggestion.sort_unstable();
+        suggestion.dedup();
+
+        let parts = suggestion
             .into_iter()
             .map(|(span, snippet)| SubstitutionPart { snippet, span })
             .collect::<Vec<_>>();
 
-        parts.sort_unstable_by_key(|part| part.span);
-
         assert!(!parts.is_empty());
         debug_assert_eq!(
             parts.iter().find(|part| part.span.is_empty() && part.snippet.is_empty()),
diff --git a/tests/ui/macros/issue-118048.rs b/tests/ui/macros/issue-118048.rs
new file mode 100644
index 00000000000..15a834fa2df
--- /dev/null
+++ b/tests/ui/macros/issue-118048.rs
@@ -0,0 +1,10 @@
+macro_rules! foo {
+    ($ty:ty) => {
+        fn foo(_: $ty, _: $ty) {}
+    }
+}
+
+foo!(_);
+//~^ ERROR the placeholder `_` is not allowed within types on item signatures for functions
+
+fn main() {}
diff --git a/tests/ui/macros/issue-118048.stderr b/tests/ui/macros/issue-118048.stderr
new file mode 100644
index 00000000000..6acf78f63b2
--- /dev/null
+++ b/tests/ui/macros/issue-118048.stderr
@@ -0,0 +1,21 @@
+error[E0121]: the placeholder `_` is not allowed within types on item signatures for functions
+  --> $DIR/issue-118048.rs:7:6
+   |
+LL | foo!(_);
+   |      ^
+   |      |
+   |      not allowed in type signatures
+   |      not allowed in type signatures
+   |
+help: use type parameters instead
+   |
+LL ~         fn foo<T>(_: $ty, _: $ty) {}
+LL |     }
+LL | }
+LL | 
+LL ~ foo!(T);
+   |
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0121`.
-- 
cgit 1.4.1-3-g733a5


From 43adf41ca6fd87cff0eb2325ffb6a68e49c6de18 Mon Sep 17 00:00:00 2001
From: Matthew Jasper <mjjasper1@gmail.com>
Date: Mon, 4 Dec 2023 16:31:13 +0000
Subject: Don't include destruction scopes in THIR

They are not used by anyone, and add memory/performance overhead.
---
 compiler/rustc_middle/src/middle/region.rs         |   4 -
 compiler/rustc_middle/src/thir.rs                  |   6 +-
 compiler/rustc_mir_build/src/build/block.rs        | 156 ++++-----
 compiler/rustc_mir_build/src/build/scope.rs        |  21 --
 compiler/rustc_mir_build/src/thir/cx/block.rs      |   6 -
 compiler/rustc_mir_build/src/thir/cx/expr.rs       |  18 +-
 compiler/rustc_mir_build/src/thir/print.rs         |  23 +-
 tests/ui/thir-print/thir-flat-const-variant.stdout |  61 ----
 tests/ui/thir-print/thir-flat.stdout               |  13 -
 tests/ui/thir-print/thir-tree-match.stdout         | 365 +++++++++------------
 tests/ui/thir-print/thir-tree.stdout               |  30 +-
 11 files changed, 235 insertions(+), 468 deletions(-)

diff --git a/compiler/rustc_middle/src/middle/region.rs b/compiler/rustc_middle/src/middle/region.rs
index b8f04ed03dc..30f13afff9a 100644
--- a/compiler/rustc_middle/src/middle/region.rs
+++ b/compiler/rustc_middle/src/middle/region.rs
@@ -349,10 +349,6 @@ impl ScopeTree {
         }
     }
 
-    pub fn opt_destruction_scope(&self, n: hir::ItemLocalId) -> Option<Scope> {
-        self.destruction_scopes.get(&n).cloned()
-    }
-
     pub fn record_var_scope(&mut self, var: hir::ItemLocalId, lifetime: Scope) {
         debug!("record_var_scope(sub={:?}, sup={:?})", var, lifetime);
         assert!(var != lifetime.item_local_id());
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index c48428c713c..b6759d35210 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -134,7 +134,6 @@ pub struct Block {
     /// This does *not* include labels on loops, e.g. `'label: loop {}`.
     pub targeted_by_break: bool,
     pub region_scope: region::Scope,
-    pub opt_destruction_scope: Option<region::Scope>,
     /// The span of the block, including the opening braces,
     /// the label, and the `unsafe` keyword, if present.
     pub span: Span,
@@ -193,7 +192,6 @@ pub enum BlockSafety {
 #[derive(Clone, Debug, HashStable)]
 pub struct Stmt<'tcx> {
     pub kind: StmtKind<'tcx>,
-    pub opt_destruction_scope: Option<region::Scope>,
 }
 
 #[derive(Clone, Debug, HashStable)]
@@ -1224,12 +1222,12 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
 mod size_asserts {
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(Block, 56);
+    static_assert_size!(Block, 48);
     static_assert_size!(Expr<'_>, 64);
     static_assert_size!(ExprKind<'_>, 40);
     static_assert_size!(Pat<'_>, 64);
     static_assert_size!(PatKind<'_>, 48);
-    static_assert_size!(Stmt<'_>, 56);
+    static_assert_size!(Stmt<'_>, 48);
     static_assert_size!(StmtKind<'_>, 48);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
index 58295b39755..8cad6976c0d 100644
--- a/compiler/rustc_mir_build/src/build/block.rs
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -13,32 +13,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         ast_block: BlockId,
         source_info: SourceInfo,
     ) -> BlockAnd<()> {
-        let Block {
-            region_scope,
-            opt_destruction_scope,
-            span,
-            ref stmts,
-            expr,
-            targeted_by_break,
-            safety_mode,
-        } = self.thir[ast_block];
+        let Block { region_scope, span, ref stmts, expr, targeted_by_break, safety_mode } =
+            self.thir[ast_block];
         let expr = expr.map(|expr| &self.thir[expr]);
-        self.in_opt_scope(opt_destruction_scope.map(|de| (de, source_info)), move |this| {
-            this.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
-                if targeted_by_break {
-                    this.in_breakable_scope(None, destination, span, |this| {
-                        Some(this.ast_block_stmts(
-                            destination,
-                            block,
-                            span,
-                            stmts,
-                            expr,
-                            safety_mode,
-                            region_scope,
-                        ))
-                    })
-                } else {
-                    this.ast_block_stmts(
+        self.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
+            if targeted_by_break {
+                this.in_breakable_scope(None, destination, span, |this| {
+                    Some(this.ast_block_stmts(
                         destination,
                         block,
                         span,
@@ -46,9 +27,19 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         expr,
                         safety_mode,
                         region_scope,
-                    )
-                }
-            })
+                    ))
+                })
+            } else {
+                this.ast_block_stmts(
+                    destination,
+                    block,
+                    span,
+                    stmts,
+                    expr,
+                    safety_mode,
+                    region_scope,
+                )
+            }
         })
     }
 
@@ -92,20 +83,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         let source_info = this.source_info(span);
         for stmt in stmts {
-            let Stmt { ref kind, opt_destruction_scope } = this.thir[*stmt];
+            let Stmt { ref kind } = this.thir[*stmt];
             match kind {
                 StmtKind::Expr { scope, expr } => {
                     this.block_context.push(BlockFrame::Statement { ignores_expr_result: true });
+                    let si = (*scope, source_info);
                     unpack!(
-                        block = this.in_opt_scope(
-                            opt_destruction_scope.map(|de| (de, source_info)),
-                            |this| {
-                                let si = (*scope, source_info);
-                                this.in_scope(si, LintLevel::Inherited, |this| {
-                                    this.stmt_expr(block, &this.thir[*expr], Some(*scope))
-                                })
-                            }
-                        )
+                        block = this.in_scope(si, LintLevel::Inherited, |this| {
+                            this.stmt_expr(block, &this.thir[*expr], Some(*scope))
+                        })
                     );
                 }
                 StmtKind::Let {
@@ -221,43 +207,38 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                     let init = &this.thir[*initializer];
                     let initializer_span = init.span;
+                    let scope = (*init_scope, source_info);
                     let failure = unpack!(
-                        block = this.in_opt_scope(
-                            opt_destruction_scope.map(|de| (de, source_info)),
-                            |this| {
-                                let scope = (*init_scope, source_info);
-                                this.in_scope(scope, *lint_level, |this| {
-                                    this.declare_bindings(
-                                        visibility_scope,
-                                        remainder_span,
-                                        pattern,
-                                        None,
-                                        Some((Some(&destination), initializer_span)),
-                                    );
-                                    this.visit_primary_bindings(
-                                        pattern,
-                                        UserTypeProjections::none(),
-                                        &mut |this, _, _, _, node, span, _, _| {
-                                            this.storage_live_binding(
-                                                block,
-                                                node,
-                                                span,
-                                                OutsideGuard,
-                                                true,
-                                            );
-                                        },
-                                    );
-                                    this.ast_let_else(
+                        block = this.in_scope(scope, *lint_level, |this| {
+                            this.declare_bindings(
+                                visibility_scope,
+                                remainder_span,
+                                pattern,
+                                None,
+                                Some((Some(&destination), initializer_span)),
+                            );
+                            this.visit_primary_bindings(
+                                pattern,
+                                UserTypeProjections::none(),
+                                &mut |this, _, _, _, node, span, _, _| {
+                                    this.storage_live_binding(
                                         block,
-                                        init,
-                                        initializer_span,
-                                        *else_block,
-                                        &last_remainder_scope,
-                                        pattern,
-                                    )
-                                })
-                            }
-                        )
+                                        node,
+                                        span,
+                                        OutsideGuard,
+                                        true,
+                                    );
+                                },
+                            );
+                            this.ast_let_else(
+                                block,
+                                init,
+                                initializer_span,
+                                *else_block,
+                                &last_remainder_scope,
+                                pattern,
+                            )
+                        })
                     );
                     this.cfg.goto(failure, source_info, failure_entry);
 
@@ -298,25 +279,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     if let Some(init) = initializer {
                         let init = &this.thir[*init];
                         let initializer_span = init.span;
+                        let scope = (*init_scope, source_info);
 
                         unpack!(
-                            block = this.in_opt_scope(
-                                opt_destruction_scope.map(|de| (de, source_info)),
-                                |this| {
-                                    let scope = (*init_scope, source_info);
-                                    this.in_scope(scope, *lint_level, |this| {
-                                        this.declare_bindings(
-                                            visibility_scope,
-                                            remainder_span,
-                                            pattern,
-                                            None,
-                                            Some((None, initializer_span)),
-                                        );
-                                        this.expr_into_pattern(block, pattern, init)
-                                        // irrefutable pattern
-                                    })
-                                },
-                            )
+                            block = this.in_scope(scope, *lint_level, |this| {
+                                this.declare_bindings(
+                                    visibility_scope,
+                                    remainder_span,
+                                    pattern,
+                                    None,
+                                    Some((None, initializer_span)),
+                                );
+                                this.expr_into_pattern(block, &pattern, init)
+                                // irrefutable pattern
+                            })
                         )
                     } else {
                         let scope = (*init_scope, source_info);
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index 993fee95895..45f417fe395 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -537,27 +537,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         (then_block, else_block)
     }
 
-    pub(crate) fn in_opt_scope<F, R>(
-        &mut self,
-        opt_scope: Option<(region::Scope, SourceInfo)>,
-        f: F,
-    ) -> BlockAnd<R>
-    where
-        F: FnOnce(&mut Builder<'a, 'tcx>) -> BlockAnd<R>,
-    {
-        debug!("in_opt_scope(opt_scope={:?})", opt_scope);
-        if let Some(region_scope) = opt_scope {
-            self.push_scope(region_scope);
-        }
-        let mut block;
-        let rv = unpack!(block = f(self));
-        if let Some(region_scope) = opt_scope {
-            unpack!(block = self.pop_scope(region_scope, block));
-        }
-        debug!("in_scope: exiting opt_scope={:?} block={:?}", opt_scope, block);
-        block.and(rv)
-    }
-
     /// Convenience wrapper that pushes a scope and then executes `f`
     /// to build its contents, popping the scope afterwards.
     #[instrument(skip(self, f), level = "debug")]
diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
index 527eadb277e..1e93e126b70 100644
--- a/compiler/rustc_mir_build/src/thir/cx/block.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -13,15 +13,12 @@ impl<'tcx> Cx<'tcx> {
         // We have to eagerly lower the "spine" of the statements
         // in order to get the lexical scoping correctly.
         let stmts = self.mirror_stmts(block.hir_id.local_id, block.stmts);
-        let opt_destruction_scope =
-            self.region_scope_tree.opt_destruction_scope(block.hir_id.local_id);
         let block = Block {
             targeted_by_break: block.targeted_by_break,
             region_scope: region::Scope {
                 id: block.hir_id.local_id,
                 data: region::ScopeData::Node,
             },
-            opt_destruction_scope,
             span: block.span,
             stmts,
             expr: block.expr.map(|expr| self.mirror_expr(expr)),
@@ -49,7 +46,6 @@ impl<'tcx> Cx<'tcx> {
             .enumerate()
             .filter_map(|(index, stmt)| {
                 let hir_id = stmt.hir_id;
-                let opt_dxn_ext = self.region_scope_tree.opt_destruction_scope(hir_id.local_id);
                 match stmt.kind {
                     hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr) => {
                         let stmt = Stmt {
@@ -60,7 +56,6 @@ impl<'tcx> Cx<'tcx> {
                                 },
                                 expr: self.mirror_expr(expr),
                             },
-                            opt_destruction_scope: opt_dxn_ext,
                         };
                         Some(self.thir.stmts.push(stmt))
                     }
@@ -122,7 +117,6 @@ impl<'tcx> Cx<'tcx> {
                                 lint_level: LintLevel::Explicit(local.hir_id),
                                 span,
                             },
-                            opt_destruction_scope: opt_dxn_ext,
                         };
                         Some(self.thir.stmts.push(stmt))
                     }
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 9759c72bf58..31874b29bb3 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -54,7 +54,7 @@ impl<'tcx> Cx<'tcx> {
 
         trace!(?expr.ty, "after adjustments");
 
-        // Next, wrap this up in the expr's scope.
+        // Finally, wrap this up in the expr's scope.
         expr = Expr {
             temp_lifetime: expr.temp_lifetime,
             ty: expr.ty,
@@ -66,22 +66,6 @@ impl<'tcx> Cx<'tcx> {
             },
         };
 
-        // Finally, create a destruction scope, if any.
-        if let Some(region_scope) =
-            self.region_scope_tree.opt_destruction_scope(hir_expr.hir_id.local_id)
-        {
-            expr = Expr {
-                temp_lifetime: expr.temp_lifetime,
-                ty: expr.ty,
-                span: hir_expr.span,
-                kind: ExprKind::Scope {
-                    region_scope,
-                    value: self.thir.exprs.push(expr),
-                    lint_level: LintLevel::Inherited,
-                },
-            };
-        }
-
         // OK, all done!
         self.thir.exprs.push(expr)
     }
diff --git a/compiler/rustc_mir_build/src/thir/print.rs b/compiler/rustc_mir_build/src/thir/print.rs
index 547da7e7007..28be3139905 100644
--- a/compiler/rustc_mir_build/src/thir/print.rs
+++ b/compiler/rustc_mir_build/src/thir/print.rs
@@ -91,23 +91,11 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
     }
 
     fn print_block(&mut self, block_id: BlockId, depth_lvl: usize) {
-        let Block {
-            targeted_by_break,
-            opt_destruction_scope,
-            span,
-            region_scope,
-            stmts,
-            expr,
-            safety_mode,
-        } = &self.thir.blocks[block_id];
+        let Block { targeted_by_break, span, region_scope, stmts, expr, safety_mode } =
+            &self.thir.blocks[block_id];
 
         print_indented!(self, "Block {", depth_lvl);
         print_indented!(self, format!("targeted_by_break: {}", targeted_by_break), depth_lvl + 1);
-        print_indented!(
-            self,
-            format!("opt_destruction_scope: {:?}", opt_destruction_scope),
-            depth_lvl + 1
-        );
         print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
         print_indented!(self, format!("region_scope: {:?}", region_scope), depth_lvl + 1);
         print_indented!(self, format!("safety_mode: {:?}", safety_mode), depth_lvl + 1);
@@ -133,14 +121,9 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
     }
 
     fn print_stmt(&mut self, stmt_id: StmtId, depth_lvl: usize) {
-        let Stmt { kind, opt_destruction_scope } = &self.thir.stmts[stmt_id];
+        let Stmt { kind } = &self.thir.stmts[stmt_id];
 
         print_indented!(self, "Stmt {", depth_lvl);
-        print_indented!(
-            self,
-            format!("opt_destruction_scope: {:?}", opt_destruction_scope),
-            depth_lvl + 1
-        );
 
         match kind {
             StmtKind::Expr { scope, expr } => {
diff --git a/tests/ui/thir-print/thir-flat-const-variant.stdout b/tests/ui/thir-print/thir-flat-const-variant.stdout
index 7bddc925996..1840be7885b 100644
--- a/tests/ui/thir-print/thir-flat-const-variant.stdout
+++ b/tests/ui/thir-print/thir-flat-const-variant.stdout
@@ -66,18 +66,6 @@ Thir {
             ),
             span: $DIR/thir-flat-const-variant.rs:12:23: 12:35 (#0),
         },
-        Expr {
-            kind: Scope {
-                region_scope: Destruction(3),
-                lint_level: Inherited,
-                value: e3,
-            },
-            ty: Foo,
-            temp_lifetime: Some(
-                Node(3),
-            ),
-            span: $DIR/thir-flat-const-variant.rs:12:23: 12:35 (#0),
-        },
     ],
     stmts: [],
     params: [],
@@ -151,18 +139,6 @@ Thir {
             ),
             span: $DIR/thir-flat-const-variant.rs:13:23: 13:36 (#0),
         },
-        Expr {
-            kind: Scope {
-                region_scope: Destruction(3),
-                lint_level: Inherited,
-                value: e3,
-            },
-            ty: Foo,
-            temp_lifetime: Some(
-                Node(3),
-            ),
-            span: $DIR/thir-flat-const-variant.rs:13:23: 13:36 (#0),
-        },
     ],
     stmts: [],
     params: [],
@@ -236,18 +212,6 @@ Thir {
             ),
             span: $DIR/thir-flat-const-variant.rs:14:24: 14:36 (#0),
         },
-        Expr {
-            kind: Scope {
-                region_scope: Destruction(3),
-                lint_level: Inherited,
-                value: e3,
-            },
-            ty: Foo,
-            temp_lifetime: Some(
-                Node(3),
-            ),
-            span: $DIR/thir-flat-const-variant.rs:14:24: 14:36 (#0),
-        },
     ],
     stmts: [],
     params: [],
@@ -321,18 +285,6 @@ Thir {
             ),
             span: $DIR/thir-flat-const-variant.rs:15:24: 15:37 (#0),
         },
-        Expr {
-            kind: Scope {
-                region_scope: Destruction(3),
-                lint_level: Inherited,
-                value: e3,
-            },
-            ty: Foo,
-            temp_lifetime: Some(
-                Node(3),
-            ),
-            span: $DIR/thir-flat-const-variant.rs:15:24: 15:37 (#0),
-        },
     ],
     stmts: [],
     params: [],
@@ -348,7 +300,6 @@ Thir {
         Block {
             targeted_by_break: false,
             region_scope: Node(1),
-            opt_destruction_scope: None,
             span: $DIR/thir-flat-const-variant.rs:18:11: 18:13 (#0),
             stmts: [],
             expr: None,
@@ -380,18 +331,6 @@ Thir {
             ),
             span: $DIR/thir-flat-const-variant.rs:18:11: 18:13 (#0),
         },
-        Expr {
-            kind: Scope {
-                region_scope: Destruction(2),
-                lint_level: Inherited,
-                value: e1,
-            },
-            ty: (),
-            temp_lifetime: Some(
-                Node(2),
-            ),
-            span: $DIR/thir-flat-const-variant.rs:18:11: 18:13 (#0),
-        },
     ],
     stmts: [],
     params: [],
diff --git a/tests/ui/thir-print/thir-flat.stdout b/tests/ui/thir-print/thir-flat.stdout
index b0aa44b56aa..a31d08adab6 100644
--- a/tests/ui/thir-print/thir-flat.stdout
+++ b/tests/ui/thir-print/thir-flat.stdout
@@ -8,7 +8,6 @@ Thir {
         Block {
             targeted_by_break: false,
             region_scope: Node(1),
-            opt_destruction_scope: None,
             span: $DIR/thir-flat.rs:4:15: 4:17 (#0),
             stmts: [],
             expr: None,
@@ -40,18 +39,6 @@ Thir {
             ),
             span: $DIR/thir-flat.rs:4:15: 4:17 (#0),
         },
-        Expr {
-            kind: Scope {
-                region_scope: Destruction(2),
-                lint_level: Inherited,
-                value: e1,
-            },
-            ty: (),
-            temp_lifetime: Some(
-                Node(2),
-            ),
-            span: $DIR/thir-flat.rs:4:15: 4:17 (#0),
-        },
     ],
     stmts: [],
     params: [],
diff --git a/tests/ui/thir-print/thir-tree-match.stdout b/tests/ui/thir-print/thir-tree-match.stdout
index 3fc130f0176..60c9283abcf 100644
--- a/tests/ui/thir-print/thir-tree-match.stdout
+++ b/tests/ui/thir-print/thir-tree-match.stdout
@@ -31,262 +31,217 @@ body:
         span: $DIR/thir-tree-match.rs:15:32: 21:2 (#0)
         kind: 
             Scope {
-                region_scope: Destruction(26)
-                lint_level: Inherited
+                region_scope: Node(26)
+                lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).26))
                 value:
                     Expr {
                         ty: bool
                         temp_lifetime: Some(Node(26))
                         span: $DIR/thir-tree-match.rs:15:32: 21:2 (#0)
                         kind: 
-                            Scope {
-                                region_scope: Node(26)
-                                lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).26))
-                                value:
+                            Block {
+                                targeted_by_break: false
+                                span: $DIR/thir-tree-match.rs:15:32: 21:2 (#0)
+                                region_scope: Node(25)
+                                safety_mode: Safe
+                                stmts: []
+                                expr:
                                     Expr {
                                         ty: bool
                                         temp_lifetime: Some(Node(26))
-                                        span: $DIR/thir-tree-match.rs:15:32: 21:2 (#0)
+                                        span: $DIR/thir-tree-match.rs:16:5: 20:6 (#0)
                                         kind: 
-                                            Block {
-                                                targeted_by_break: false
-                                                opt_destruction_scope: None
-                                                span: $DIR/thir-tree-match.rs:15:32: 21:2 (#0)
-                                                region_scope: Node(25)
-                                                safety_mode: Safe
-                                                stmts: []
-                                                expr:
+                                            Scope {
+                                                region_scope: Node(3)
+                                                lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).3))
+                                                value:
                                                     Expr {
                                                         ty: bool
                                                         temp_lifetime: Some(Node(26))
                                                         span: $DIR/thir-tree-match.rs:16:5: 20:6 (#0)
                                                         kind: 
-                                                            Scope {
-                                                                region_scope: Node(3)
-                                                                lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).3))
-                                                                value:
+                                                            Match {
+                                                                scrutinee:
                                                                     Expr {
-                                                                        ty: bool
+                                                                        ty: Foo
                                                                         temp_lifetime: Some(Node(26))
-                                                                        span: $DIR/thir-tree-match.rs:16:5: 20:6 (#0)
+                                                                        span: $DIR/thir-tree-match.rs:16:11: 16:14 (#0)
                                                                         kind: 
-                                                                            Match {
-                                                                                scrutinee:
+                                                                            Scope {
+                                                                                region_scope: Node(4)
+                                                                                lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).4))
+                                                                                value:
                                                                                     Expr {
                                                                                         ty: Foo
                                                                                         temp_lifetime: Some(Node(26))
                                                                                         span: $DIR/thir-tree-match.rs:16:11: 16:14 (#0)
                                                                                         kind: 
-                                                                                            Scope {
-                                                                                                region_scope: Node(4)
-                                                                                                lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).4))
-                                                                                                value:
-                                                                                                    Expr {
-                                                                                                        ty: Foo
-                                                                                                        temp_lifetime: Some(Node(26))
-                                                                                                        span: $DIR/thir-tree-match.rs:16:11: 16:14 (#0)
-                                                                                                        kind: 
-                                                                                                            VarRef {
-                                                                                                                id: LocalVarId(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).2))
-                                                                                                            }
-                                                                                                    }
+                                                                                            VarRef {
+                                                                                                id: LocalVarId(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).2))
                                                                                             }
                                                                                     }
-                                                                                arms: [
-                                                                                    Arm {
-                                                                                        pattern: 
+                                                                            }
+                                                                    }
+                                                                arms: [
+                                                                    Arm {
+                                                                        pattern: 
+                                                                            Pat: {
+                                                                                ty: Foo
+                                                                                span: $DIR/thir-tree-match.rs:17:9: 17:32 (#0)
+                                                                                kind: PatKind {
+                                                                                    Variant {
+                                                                                        adt_def: 
+                                                                                            AdtDef {
+                                                                                                did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo)
+                                                                                                variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }]
+                                                                                                flags: IS_ENUM
+                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 }
+                                                                                        args: []
+                                                                                        variant_index: 0
+                                                                                        subpatterns: [
                                                                                             Pat: {
-                                                                                                ty: Foo
-                                                                                                span: $DIR/thir-tree-match.rs:17:9: 17:32 (#0)
+                                                                                                ty: Bar
+                                                                                                span: $DIR/thir-tree-match.rs:17:21: 17:31 (#0)
                                                                                                 kind: PatKind {
                                                                                                     Variant {
                                                                                                         adt_def: 
                                                                                                             AdtDef {
-                                                                                                                did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo)
-                                                                                                                variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }]
+                                                                                                                did: DefId(0:3 ~ thir_tree_match[fcf8]::Bar)
+                                                                                                                variants: [VariantDef { def_id: DefId(0:4 ~ thir_tree_match[fcf8]::Bar::First), ctor: Some((Const, DefId(0:5 ~ thir_tree_match[fcf8]::Bar::First::{constructor#0}))), name: "First", discr: Relative(0), fields: [], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:6 ~ thir_tree_match[fcf8]::Bar::Second), ctor: Some((Const, DefId(0:7 ~ thir_tree_match[fcf8]::Bar::Second::{constructor#0}))), name: "Second", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:8 ~ thir_tree_match[fcf8]::Bar::Third), ctor: Some((Const, DefId(0:9 ~ thir_tree_match[fcf8]::Bar::Third::{constructor#0}))), name: "Third", discr: Relative(2), fields: [], flags: NO_VARIANT_FLAGS }]
                                                                                                                 flags: IS_ENUM
-                                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 }
+                                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 10333377570083945360 }
                                                                                                         args: []
                                                                                                         variant_index: 0
-                                                                                                        subpatterns: [
-                                                                                                            Pat: {
-                                                                                                                ty: Bar
-                                                                                                                span: $DIR/thir-tree-match.rs:17:21: 17:31 (#0)
-                                                                                                                kind: PatKind {
-                                                                                                                    Variant {
-                                                                                                                        adt_def: 
-                                                                                                                            AdtDef {
-                                                                                                                                did: DefId(0:3 ~ thir_tree_match[fcf8]::Bar)
-                                                                                                                                variants: [VariantDef { def_id: DefId(0:4 ~ thir_tree_match[fcf8]::Bar::First), ctor: Some((Const, DefId(0:5 ~ thir_tree_match[fcf8]::Bar::First::{constructor#0}))), name: "First", discr: Relative(0), fields: [], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:6 ~ thir_tree_match[fcf8]::Bar::Second), ctor: Some((Const, DefId(0:7 ~ thir_tree_match[fcf8]::Bar::Second::{constructor#0}))), name: "Second", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:8 ~ thir_tree_match[fcf8]::Bar::Third), ctor: Some((Const, DefId(0:9 ~ thir_tree_match[fcf8]::Bar::Third::{constructor#0}))), name: "Third", discr: Relative(2), fields: [], flags: NO_VARIANT_FLAGS }]
-                                                                                                                                flags: IS_ENUM
-                                                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 10333377570083945360 }
-                                                                                                                        args: []
-                                                                                                                        variant_index: 0
-                                                                                                                        subpatterns: []
-                                                                                                                    }
-                                                                                                                }
-                                                                                                            }
-                                                                                                        ]
+                                                                                                        subpatterns: []
                                                                                                     }
                                                                                                 }
                                                                                             }
-                                                                                        guard: None
-                                                                                        body: 
+                                                                                        ]
+                                                                                    }
+                                                                                }
+                                                                            }
+                                                                        guard: None
+                                                                        body: 
+                                                                            Expr {
+                                                                                ty: bool
+                                                                                temp_lifetime: Some(Node(13))
+                                                                                span: $DIR/thir-tree-match.rs:17:36: 17:40 (#0)
+                                                                                kind: 
+                                                                                    Scope {
+                                                                                        region_scope: Node(13)
+                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).13))
+                                                                                        value:
                                                                                             Expr {
                                                                                                 ty: bool
                                                                                                 temp_lifetime: Some(Node(13))
                                                                                                 span: $DIR/thir-tree-match.rs:17:36: 17:40 (#0)
                                                                                                 kind: 
-                                                                                                    Scope {
-                                                                                                        region_scope: Destruction(13)
-                                                                                                        lint_level: Inherited
-                                                                                                        value:
-                                                                                                            Expr {
-                                                                                                                ty: bool
-                                                                                                                temp_lifetime: Some(Node(13))
-                                                                                                                span: $DIR/thir-tree-match.rs:17:36: 17:40 (#0)
-                                                                                                                kind: 
-                                                                                                                    Scope {
-                                                                                                                        region_scope: Node(13)
-                                                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).13))
-                                                                                                                        value:
-                                                                                                                            Expr {
-                                                                                                                                ty: bool
-                                                                                                                                temp_lifetime: Some(Node(13))
-                                                                                                                                span: $DIR/thir-tree-match.rs:17:36: 17:40 (#0)
-                                                                                                                                kind: 
-                                                                                                                                    Literal( lit: Spanned { node: Bool(true), span: $DIR/thir-tree-match.rs:17:36: 17:40 (#0) }, neg: false)
+                                                                                                    Literal( lit: Spanned { node: Bool(true), span: $DIR/thir-tree-match.rs:17:36: 17:40 (#0) }, neg: false)
 
-                                                                                                                            }
-                                                                                                                    }
-                                                                                                            }
-                                                                                                    }
                                                                                             }
-                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).12))
-                                                                                        scope: Node(12)
-                                                                                        span: $DIR/thir-tree-match.rs:17:9: 17:40 (#0)
                                                                                     }
-                                                                                    Arm {
-                                                                                        pattern: 
+                                                                            }
+                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).12))
+                                                                        scope: Node(12)
+                                                                        span: $DIR/thir-tree-match.rs:17:9: 17:40 (#0)
+                                                                    }
+                                                                    Arm {
+                                                                        pattern: 
+                                                                            Pat: {
+                                                                                ty: Foo
+                                                                                span: $DIR/thir-tree-match.rs:18:9: 18:23 (#0)
+                                                                                kind: PatKind {
+                                                                                    Variant {
+                                                                                        adt_def: 
+                                                                                            AdtDef {
+                                                                                                did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo)
+                                                                                                variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }]
+                                                                                                flags: IS_ENUM
+                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 }
+                                                                                        args: []
+                                                                                        variant_index: 0
+                                                                                        subpatterns: [
                                                                                             Pat: {
-                                                                                                ty: Foo
-                                                                                                span: $DIR/thir-tree-match.rs:18:9: 18:23 (#0)
+                                                                                                ty: Bar
+                                                                                                span: $DIR/thir-tree-match.rs:18:21: 18:22 (#0)
                                                                                                 kind: PatKind {
-                                                                                                    Variant {
-                                                                                                        adt_def: 
-                                                                                                            AdtDef {
-                                                                                                                did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo)
-                                                                                                                variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }]
-                                                                                                                flags: IS_ENUM
-                                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 }
-                                                                                                        args: []
-                                                                                                        variant_index: 0
-                                                                                                        subpatterns: [
-                                                                                                            Pat: {
-                                                                                                                ty: Bar
-                                                                                                                span: $DIR/thir-tree-match.rs:18:21: 18:22 (#0)
-                                                                                                                kind: PatKind {
-                                                                                                                    Wild
-                                                                                                                }
-                                                                                                            }
-                                                                                                        ]
-                                                                                                    }
+                                                                                                    Wild
                                                                                                 }
                                                                                             }
-                                                                                        guard: None
-                                                                                        body: 
+                                                                                        ]
+                                                                                    }
+                                                                                }
+                                                                            }
+                                                                        guard: None
+                                                                        body: 
+                                                                            Expr {
+                                                                                ty: bool
+                                                                                temp_lifetime: Some(Node(19))
+                                                                                span: $DIR/thir-tree-match.rs:18:27: 18:32 (#0)
+                                                                                kind: 
+                                                                                    Scope {
+                                                                                        region_scope: Node(19)
+                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).19))
+                                                                                        value:
                                                                                             Expr {
                                                                                                 ty: bool
                                                                                                 temp_lifetime: Some(Node(19))
                                                                                                 span: $DIR/thir-tree-match.rs:18:27: 18:32 (#0)
                                                                                                 kind: 
-                                                                                                    Scope {
-                                                                                                        region_scope: Destruction(19)
-                                                                                                        lint_level: Inherited
-                                                                                                        value:
-                                                                                                            Expr {
-                                                                                                                ty: bool
-                                                                                                                temp_lifetime: Some(Node(19))
-                                                                                                                span: $DIR/thir-tree-match.rs:18:27: 18:32 (#0)
-                                                                                                                kind: 
-                                                                                                                    Scope {
-                                                                                                                        region_scope: Node(19)
-                                                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).19))
-                                                                                                                        value:
-                                                                                                                            Expr {
-                                                                                                                                ty: bool
-                                                                                                                                temp_lifetime: Some(Node(19))
-                                                                                                                                span: $DIR/thir-tree-match.rs:18:27: 18:32 (#0)
-                                                                                                                                kind: 
-                                                                                                                                    Literal( lit: Spanned { node: Bool(false), span: $DIR/thir-tree-match.rs:18:27: 18:32 (#0) }, neg: false)
+                                                                                                    Literal( lit: Spanned { node: Bool(false), span: $DIR/thir-tree-match.rs:18:27: 18:32 (#0) }, neg: false)
 
-                                                                                                                            }
-                                                                                                                    }
-                                                                                                            }
-                                                                                                    }
                                                                                             }
-                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).18))
-                                                                                        scope: Node(18)
-                                                                                        span: $DIR/thir-tree-match.rs:18:9: 18:32 (#0)
                                                                                     }
-                                                                                    Arm {
-                                                                                        pattern: 
-                                                                                            Pat: {
-                                                                                                ty: Foo
-                                                                                                span: $DIR/thir-tree-match.rs:19:9: 19:20 (#0)
-                                                                                                kind: PatKind {
-                                                                                                    Variant {
-                                                                                                        adt_def: 
-                                                                                                            AdtDef {
-                                                                                                                did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo)
-                                                                                                                variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }]
-                                                                                                                flags: IS_ENUM
-                                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 }
-                                                                                                        args: []
-                                                                                                        variant_index: 1
-                                                                                                        subpatterns: []
-                                                                                                    }
-                                                                                                }
-                                                                                            }
-                                                                                        guard: None
-                                                                                        body: 
+                                                                            }
+                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).18))
+                                                                        scope: Node(18)
+                                                                        span: $DIR/thir-tree-match.rs:18:9: 18:32 (#0)
+                                                                    }
+                                                                    Arm {
+                                                                        pattern: 
+                                                                            Pat: {
+                                                                                ty: Foo
+                                                                                span: $DIR/thir-tree-match.rs:19:9: 19:20 (#0)
+                                                                                kind: PatKind {
+                                                                                    Variant {
+                                                                                        adt_def: 
+                                                                                            AdtDef {
+                                                                                                did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo)
+                                                                                                variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }]
+                                                                                                flags: IS_ENUM
+                                                                                                repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 }
+                                                                                        args: []
+                                                                                        variant_index: 1
+                                                                                        subpatterns: []
+                                                                                    }
+                                                                                }
+                                                                            }
+                                                                        guard: None
+                                                                        body: 
+                                                                            Expr {
+                                                                                ty: bool
+                                                                                temp_lifetime: Some(Node(24))
+                                                                                span: $DIR/thir-tree-match.rs:19:24: 19:28 (#0)
+                                                                                kind: 
+                                                                                    Scope {
+                                                                                        region_scope: Node(24)
+                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).24))
+                                                                                        value:
                                                                                             Expr {
                                                                                                 ty: bool
                                                                                                 temp_lifetime: Some(Node(24))
                                                                                                 span: $DIR/thir-tree-match.rs:19:24: 19:28 (#0)
                                                                                                 kind: 
-                                                                                                    Scope {
-                                                                                                        region_scope: Destruction(24)
-                                                                                                        lint_level: Inherited
-                                                                                                        value:
-                                                                                                            Expr {
-                                                                                                                ty: bool
-                                                                                                                temp_lifetime: Some(Node(24))
-                                                                                                                span: $DIR/thir-tree-match.rs:19:24: 19:28 (#0)
-                                                                                                                kind: 
-                                                                                                                    Scope {
-                                                                                                                        region_scope: Node(24)
-                                                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).24))
-                                                                                                                        value:
-                                                                                                                            Expr {
-                                                                                                                                ty: bool
-                                                                                                                                temp_lifetime: Some(Node(24))
-                                                                                                                                span: $DIR/thir-tree-match.rs:19:24: 19:28 (#0)
-                                                                                                                                kind: 
-                                                                                                                                    Literal( lit: Spanned { node: Bool(true), span: $DIR/thir-tree-match.rs:19:24: 19:28 (#0) }, neg: false)
+                                                                                                    Literal( lit: Spanned { node: Bool(true), span: $DIR/thir-tree-match.rs:19:24: 19:28 (#0) }, neg: false)
 
-                                                                                                                            }
-                                                                                                                    }
-                                                                                                            }
-                                                                                                    }
                                                                                             }
-                                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).23))
-                                                                                        scope: Node(23)
-                                                                                        span: $DIR/thir-tree-match.rs:19:9: 19:28 (#0)
                                                                                     }
-                                                                                ]
                                                                             }
+                                                                        lint_level: Explicit(HirId(DefId(0:16 ~ thir_tree_match[fcf8]::has_match).23))
+                                                                        scope: Node(23)
+                                                                        span: $DIR/thir-tree-match.rs:19:9: 19:28 (#0)
                                                                     }
+                                                                ]
                                                             }
                                                     }
                                             }
@@ -307,33 +262,21 @@ body:
         span: $DIR/thir-tree-match.rs:23:11: 23:13 (#0)
         kind: 
             Scope {
-                region_scope: Destruction(2)
-                lint_level: Inherited
+                region_scope: Node(2)
+                lint_level: Explicit(HirId(DefId(0:17 ~ thir_tree_match[fcf8]::main).2))
                 value:
                     Expr {
                         ty: ()
                         temp_lifetime: Some(Node(2))
                         span: $DIR/thir-tree-match.rs:23:11: 23:13 (#0)
                         kind: 
-                            Scope {
-                                region_scope: Node(2)
-                                lint_level: Explicit(HirId(DefId(0:17 ~ thir_tree_match[fcf8]::main).2))
-                                value:
-                                    Expr {
-                                        ty: ()
-                                        temp_lifetime: Some(Node(2))
-                                        span: $DIR/thir-tree-match.rs:23:11: 23:13 (#0)
-                                        kind: 
-                                            Block {
-                                                targeted_by_break: false
-                                                opt_destruction_scope: None
-                                                span: $DIR/thir-tree-match.rs:23:11: 23:13 (#0)
-                                                region_scope: Node(1)
-                                                safety_mode: Safe
-                                                stmts: []
-                                                expr: []
-                                            }
-                                    }
+                            Block {
+                                targeted_by_break: false
+                                span: $DIR/thir-tree-match.rs:23:11: 23:13 (#0)
+                                region_scope: Node(1)
+                                safety_mode: Safe
+                                stmts: []
+                                expr: []
                             }
                     }
             }
diff --git a/tests/ui/thir-print/thir-tree.stdout b/tests/ui/thir-print/thir-tree.stdout
index 1b478dbef99..ef6db368dbe 100644
--- a/tests/ui/thir-print/thir-tree.stdout
+++ b/tests/ui/thir-print/thir-tree.stdout
@@ -8,33 +8,21 @@ body:
         span: $DIR/thir-tree.rs:4:15: 4:17 (#0)
         kind: 
             Scope {
-                region_scope: Destruction(2)
-                lint_level: Inherited
+                region_scope: Node(2)
+                lint_level: Explicit(HirId(DefId(0:3 ~ thir_tree[7aaa]::main).2))
                 value:
                     Expr {
                         ty: ()
                         temp_lifetime: Some(Node(2))
                         span: $DIR/thir-tree.rs:4:15: 4:17 (#0)
                         kind: 
-                            Scope {
-                                region_scope: Node(2)
-                                lint_level: Explicit(HirId(DefId(0:3 ~ thir_tree[7aaa]::main).2))
-                                value:
-                                    Expr {
-                                        ty: ()
-                                        temp_lifetime: Some(Node(2))
-                                        span: $DIR/thir-tree.rs:4:15: 4:17 (#0)
-                                        kind: 
-                                            Block {
-                                                targeted_by_break: false
-                                                opt_destruction_scope: None
-                                                span: $DIR/thir-tree.rs:4:15: 4:17 (#0)
-                                                region_scope: Node(1)
-                                                safety_mode: Safe
-                                                stmts: []
-                                                expr: []
-                                            }
-                                    }
+                            Block {
+                                targeted_by_break: false
+                                span: $DIR/thir-tree.rs:4:15: 4:17 (#0)
+                                region_scope: Node(1)
+                                safety_mode: Safe
+                                stmts: []
+                                expr: []
                             }
                     }
             }
-- 
cgit 1.4.1-3-g733a5


From 3f8487a0996de85b611b7c10fd5a3f60b10473a6 Mon Sep 17 00:00:00 2001
From: l00846161 <liuxinyi27@huawei.com>
Date: Thu, 16 Nov 2023 14:17:54 +0800
Subject: Add safe compilation options

Add two options when building rust: strip and stack protector.
If set `strip = true`, symbols will be stripped using `-Cstrip=symbols`.
Also can set `stack-protector` and stack protectors will be used.
---
 config.example.toml                     | 10 ++++++++++
 src/bootstrap/src/core/builder.rs       |  6 ++++++
 src/bootstrap/src/core/config/config.rs |  7 +++++++
 3 files changed, 23 insertions(+)

diff --git a/config.example.toml b/config.example.toml
index c91222169d9..49c4ad4c958 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -600,6 +600,16 @@ change-id = 117813
 # desired in distributions, for example.
 #rpath = true
 
+# Indicates whether symbols should be stripped using `-Cstrip=symbols`.
+#strip = false
+
+# Indicates whether stack protectors should be used
+# via the unstable option `-Zstack-protector`.
+#
+# Valid options are : `none`(default),`basic`,`strong`, or `all`.
+# `strong` and `basic` options may be buggy and are not recommended, see rust-lang/rust#114903.
+#stack-protector = "none"
+
 # Prints each test name as it is executed, to help debug issues in the test harness itself.
 #verbose-tests = false
 
diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs
index 65af2aed6de..8c73a2ad5c1 100644
--- a/src/bootstrap/src/core/builder.rs
+++ b/src/bootstrap/src/core/builder.rs
@@ -1667,6 +1667,12 @@ impl<'a> Builder<'a> {
             }
         }
 
+        cargo.env(profile_var("STRIP"), self.config.rust_strip.to_string());
+
+        if let Some(stack_protector) = &self.config.rust_stack_protector {
+            rustflags.arg(&format!("-Zstack-protector={stack_protector}"));
+        }
+
         if let Some(host_linker) = self.linker(compiler.host) {
             hostflags.arg(format!("-Clinker={}", host_linker.display()));
         }
diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index 22e8ce8365b..1527cc3e46a 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -222,6 +222,8 @@ pub struct Config {
     pub rust_debuginfo_level_tests: DebuginfoLevel,
     pub rust_split_debuginfo: SplitDebuginfo,
     pub rust_rpath: bool,
+    pub rust_strip: bool,
+    pub rust_stack_protector: Option<String>,
     pub rustc_parallel: bool,
     pub rustc_default_linker: Option<String>,
     pub rust_optimize_tests: bool,
@@ -1001,6 +1003,8 @@ define_config! {
         description: Option<String> = "description",
         musl_root: Option<String> = "musl-root",
         rpath: Option<bool> = "rpath",
+        strip: Option<bool> = "strip",
+        stack_protector: Option<String> = "stack-protector",
         verbose_tests: Option<bool> = "verbose-tests",
         optimize_tests: Option<bool> = "optimize-tests",
         codegen_tests: Option<bool> = "codegen-tests",
@@ -1069,6 +1073,7 @@ impl Config {
         config.docs = true;
         config.docs_minification = true;
         config.rust_rpath = true;
+        config.rust_strip = false;
         config.channel = "dev".to_string();
         config.codegen_tests = true;
         config.rust_dist_src = true;
@@ -1422,6 +1427,8 @@ impl Config {
             set(&mut config.rust_optimize_tests, rust.optimize_tests);
             set(&mut config.codegen_tests, rust.codegen_tests);
             set(&mut config.rust_rpath, rust.rpath);
+            set(&mut config.rust_strip, rust.strip);
+            config.rust_stack_protector = rust.stack_protector;
             set(&mut config.jemalloc, rust.jemalloc);
             set(&mut config.test_compare_mode, rust.test_compare_mode);
             set(&mut config.backtrace, rust.backtrace);
-- 
cgit 1.4.1-3-g733a5


From e5b76892cc5e6fbfb495bf6a7339962a287849a7 Mon Sep 17 00:00:00 2001
From: quininer <quininer@live.com>
Date: Mon, 13 Nov 2023 20:48:23 +0800
Subject: Add emulated TLS support

Currently LLVM uses emutls by default
for some targets (such as android, openbsd),
but rust does not use it, because `has_thread_local` is false.

This commit has some changes to allow users to enable emutls:

1. add `-Zhas-thread-local` flag to specify
    that std uses `#[thread_local]` instead of pthread key.
2. when using emutls, decorate symbol names
    to find thread local symbol correctly.
3. change `-Zforce-emulated-tls` to `-Ztls-model=emulated`
    to explicitly specify whether to generate emutls.
---
 compiler/rustc_codegen_gcc/src/context.rs          |  1 +
 .../src/back/owned_target_machine.rs               |  4 +--
 compiler/rustc_codegen_llvm/src/back/write.rs      |  6 ++--
 compiler/rustc_codegen_llvm/src/context.rs         |  1 +
 compiler/rustc_codegen_llvm/src/lib.rs             |  4 ++-
 compiler/rustc_codegen_llvm/src/llvm/ffi.rs        |  2 +-
 compiler/rustc_codegen_ssa/src/back/linker.rs      |  4 ++-
 .../rustc_codegen_ssa/src/back/symbol_export.rs    | 34 +++++++++++++++++++++-
 compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp   | 10 ++-----
 compiler/rustc_session/src/config.rs               |  2 +-
 compiler/rustc_session/src/options.rs              |  2 ++
 compiler/rustc_target/src/spec/base/android.rs     |  3 +-
 compiler/rustc_target/src/spec/base/linux_ohos.rs  |  4 +--
 compiler/rustc_target/src/spec/base/openbsd.rs     |  3 +-
 compiler/rustc_target/src/spec/mod.rs              |  9 ++----
 library/std/src/sys/unix/thread_local_dtor.rs      |  8 ++++-
 .../unstable-book/src/compiler-flags/tls-model.md  |  2 ++
 17 files changed, 71 insertions(+), 28 deletions(-)

diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs
index a043660ea63..893cad05161 100644
--- a/compiler/rustc_codegen_gcc/src/context.rs
+++ b/compiler/rustc_codegen_gcc/src/context.rs
@@ -569,5 +569,6 @@ fn to_gcc_tls_mode(tls_model: TlsModel) -> gccjit::TlsModel {
         TlsModel::LocalDynamic => gccjit::TlsModel::LocalDynamic,
         TlsModel::InitialExec => gccjit::TlsModel::InitialExec,
         TlsModel::LocalExec => gccjit::TlsModel::LocalExec,
+        TlsModel::Emulated => gccjit::TlsModel::GlobalDynamic,
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
index 36484c3c3fc..28a88dd2efe 100644
--- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
+++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
@@ -39,7 +39,7 @@ impl OwnedTargetMachine {
         split_dwarf_file: &CStr,
         output_obj_file: &CStr,
         debug_info_compression: &CStr,
-        force_emulated_tls: bool,
+        use_emulated_tls: bool,
         args_cstr_buff: &[u8],
     ) -> Result<Self, LlvmError<'static>> {
         assert!(args_cstr_buff.len() > 0);
@@ -71,7 +71,7 @@ impl OwnedTargetMachine {
                 split_dwarf_file.as_ptr(),
                 output_obj_file.as_ptr(),
                 debug_info_compression.as_ptr(),
-                force_emulated_tls,
+                use_emulated_tls,
                 args_cstr_buff.as_ptr() as *const c_char,
                 args_cstr_buff.len(),
             )
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index 1a567c0fce8..bdabb9129a7 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -33,7 +33,7 @@ use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, Switc
 use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
-use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo};
+use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
 
 use crate::llvm::diagnostic::OptimizationDiagnosticKind;
 use libc::{c_char, c_int, c_uint, c_void, size_t};
@@ -223,7 +223,7 @@ pub fn target_machine_factory(
 
     let path_mapping = sess.source_map().path_mapping().clone();
 
-    let force_emulated_tls = sess.target.force_emulated_tls;
+    let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
 
     // copy the exe path, followed by path all into one buffer
     // null terminating them so we can use them as null terminated strings
@@ -297,7 +297,7 @@ pub fn target_machine_factory(
             &split_dwarf_file,
             &output_obj_file,
             &debuginfo_compression,
-            force_emulated_tls,
+            use_emulated_tls,
             &args_cstr_buff,
         )
     })
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 92a8c00510b..ed0d5e68cbf 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -120,6 +120,7 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode {
         TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic,
         TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec,
         TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec,
+        TlsModel::Emulated => llvm::ThreadLocalMode::GeneralDynamic,
     }
 }
 
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index 915cf31de08..33b19ab362a 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -306,7 +306,9 @@ impl CodegenBackend for LlvmCodegenBackend {
             }
             PrintKind::TlsModels => {
                 writeln!(out, "Available TLS models:");
-                for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] {
+                for name in
+                    &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"]
+                {
                     writeln!(out, "    {name}");
                 }
                 writeln!(out);
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 6c3ccc9cf0d..432cfe203c8 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -2159,7 +2159,7 @@ extern "C" {
         SplitDwarfFile: *const c_char,
         OutputObjFile: *const c_char,
         DebugInfoCompression: *const c_char,
-        ForceEmulatedTls: bool,
+        UseEmulatedTls: bool,
         ArgsCstrBuff: *const c_char,
         ArgsCstrBuffLen: usize,
     ) -> *mut TargetMachine;
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index bb5f6e27e4d..eeb57d4d0f4 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -1748,7 +1748,9 @@ fn exported_symbols_for_non_proc_macro(tcx: TyCtxt<'_>, crate_type: CrateType) -
     let export_threshold = symbol_export::crates_export_threshold(&[crate_type]);
     for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| {
         if info.level.is_below_threshold(export_threshold) {
-            symbols.push(symbol_export::symbol_name_for_instance_in_crate(tcx, symbol, cnum));
+            symbols.push(symbol_export::exporting_symbol_name_for_instance_in_crate(
+                tcx, symbol, cnum,
+            ));
         }
     });
 
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 5f2fad0536b..c092ce0fddf 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -16,7 +16,7 @@ use rustc_middle::ty::{self, SymbolName, TyCtxt};
 use rustc_middle::ty::{GenericArgKind, GenericArgsRef};
 use rustc_middle::util::Providers;
 use rustc_session::config::{CrateType, OomStrategy};
-use rustc_target::spec::SanitizerSet;
+use rustc_target::spec::{SanitizerSet, TlsModel};
 
 pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel {
     crates_export_threshold(tcx.crate_types())
@@ -552,6 +552,12 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
 
     let mut undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate);
 
+    // thread local will not be a function call,
+    // so it is safe to return before windows symbol decoration check.
+    if let Some(name) = maybe_emutls_symbol_name(tcx, symbol, &undecorated) {
+        return name;
+    }
+
     let target = &tcx.sess.target;
     if !target.is_like_windows {
         // Mach-O has a global "_" suffix and `object` crate will handle it.
@@ -612,6 +618,32 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
     format!("{prefix}{undecorated}{suffix}{args_in_bytes}")
 }
 
+pub fn exporting_symbol_name_for_instance_in_crate<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    symbol: ExportedSymbol<'tcx>,
+    cnum: CrateNum,
+) -> String {
+    let undecorated = symbol_name_for_instance_in_crate(tcx, symbol, cnum);
+    maybe_emutls_symbol_name(tcx, symbol, &undecorated).unwrap_or(undecorated)
+}
+
+fn maybe_emutls_symbol_name<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    symbol: ExportedSymbol<'tcx>,
+    undecorated: &str,
+) -> Option<String> {
+    if matches!(tcx.sess.tls_model(), TlsModel::Emulated)
+        && let ExportedSymbol::NonGeneric(def_id) = symbol
+        && tcx.is_thread_local_static(def_id)
+    {
+        // When using emutls, LLVM will add the `__emutls_v.` prefix to thread local symbols,
+        // and exported symbol name need to match this.
+        Some(format!("__emutls_v.{undecorated}"))
+    } else {
+        None
+    }
+}
+
 fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> FxHashMap<DefId, String> {
     // Build up a map from DefId to a `NativeLib` structure, where
     // `NativeLib` internally contains information about
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
index 0edcac93b62..55e1c84c8a2 100644
--- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
@@ -410,7 +410,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     const char *SplitDwarfFile,
     const char *OutputObjFile,
     const char *DebugInfoCompression,
-    bool ForceEmulatedTls,
+    bool UseEmulatedTls,
     const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
 
   auto OptLevel = fromRust(RustOptLevel);
@@ -456,13 +456,9 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   Options.UseInitArray = UseInitArray;
 
 #if LLVM_VERSION_LT(17, 0)
-  if (ForceEmulatedTls) {
-    Options.ExplicitEmulatedTLS = true;
-    Options.EmulatedTLS = true;
-  }
-#else
-  Options.EmulatedTLS = ForceEmulatedTls || Trip.hasDefaultEmulatedTLS();
+  Options.ExplicitEmulatedTLS = true;
 #endif
+  Options.EmulatedTLS = UseEmulatedTls;
 
   if (TrapUnreachable) {
     // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index e694e150b31..edf12023af1 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -1283,7 +1283,7 @@ fn default_configuration(sess: &Session) -> Cfg {
         ret.insert((sym::relocation_model, Some(relocation_model)));
     }
     ret.insert((sym::target_vendor, Some(Symbol::intern(vendor))));
-    if sess.target.has_thread_local {
+    if sess.opts.unstable_opts.has_thread_local.unwrap_or(sess.target.has_thread_local) {
         ret.insert((sym::target_thread_local, None));
     }
     let mut has_atomic = false;
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 7a6108bfbe2..b1cf43f471a 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1624,6 +1624,8 @@ options! {
     graphviz_font: String = ("Courier, monospace".to_string(), parse_string, [UNTRACKED],
         "use the given `fontname` in graphviz output; can be overridden by setting \
         environment variable `RUSTC_GRAPHVIZ_FONT` (default: `Courier, monospace`)"),
+    has_thread_local: Option<bool> = (None, parse_opt_bool, [TRACKED],
+        "explicitly enable the `cfg(target_thread_local)` directive"),
     hir_stats: bool = (false, parse_bool, [UNTRACKED],
         "print some statistics about AST and HIR (default: no)"),
     human_readable_cgu_names: bool = (false, parse_bool, [TRACKED],
diff --git a/compiler/rustc_target/src/spec/base/android.rs b/compiler/rustc_target/src/spec/base/android.rs
index af15c16a5a9..5320f1b4bbb 100644
--- a/compiler/rustc_target/src/spec/base/android.rs
+++ b/compiler/rustc_target/src/spec/base/android.rs
@@ -1,10 +1,11 @@
-use crate::spec::{base, SanitizerSet, TargetOptions};
+use crate::spec::{base, SanitizerSet, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     let mut base = base::linux::opts();
     base.os = "android".into();
     base.is_like_android = true;
     base.default_dwarf_version = 2;
+    base.tls_model = TlsModel::Emulated;
     base.has_thread_local = false;
     base.supported_sanitizers = SanitizerSet::ADDRESS;
     // This is for backward compatibility, see https://github.com/rust-lang/rust/issues/49867
diff --git a/compiler/rustc_target/src/spec/base/linux_ohos.rs b/compiler/rustc_target/src/spec/base/linux_ohos.rs
index 273e6a98dd4..8272cda05e7 100644
--- a/compiler/rustc_target/src/spec/base/linux_ohos.rs
+++ b/compiler/rustc_target/src/spec/base/linux_ohos.rs
@@ -1,11 +1,11 @@
-use crate::spec::{base, TargetOptions};
+use crate::spec::{base, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     let mut base = base::linux::opts();
 
     base.env = "ohos".into();
     base.crt_static_default = false;
-    base.force_emulated_tls = true;
+    base.tls_model = TlsModel::Emulated;
     base.has_thread_local = false;
 
     base
diff --git a/compiler/rustc_target/src/spec/base/openbsd.rs b/compiler/rustc_target/src/spec/base/openbsd.rs
index e7db14e05a4..bc3aecb5c1e 100644
--- a/compiler/rustc_target/src/spec/base/openbsd.rs
+++ b/compiler/rustc_target/src/spec/base/openbsd.rs
@@ -1,4 +1,4 @@
-use crate::spec::{cvs, FramePointer, RelroLevel, TargetOptions};
+use crate::spec::{cvs, FramePointer, RelroLevel, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     TargetOptions {
@@ -11,6 +11,7 @@ pub fn opts() -> TargetOptions {
         frame_pointer: FramePointer::Always, // FIXME 43575: should be MayOmit...
         relro_level: RelroLevel::Full,
         default_dwarf_version: 2,
+        tls_model: TlsModel::Emulated,
         ..Default::default()
     }
 }
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index dbce2f30f93..def74d37df7 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -929,6 +929,7 @@ pub enum TlsModel {
     LocalDynamic,
     InitialExec,
     LocalExec,
+    Emulated,
 }
 
 impl FromStr for TlsModel {
@@ -942,6 +943,7 @@ impl FromStr for TlsModel {
             "local-dynamic" => TlsModel::LocalDynamic,
             "initial-exec" => TlsModel::InitialExec,
             "local-exec" => TlsModel::LocalExec,
+            "emulated" => TlsModel::Emulated,
             _ => return Err(()),
         })
     }
@@ -954,6 +956,7 @@ impl ToJson for TlsModel {
             TlsModel::LocalDynamic => "local-dynamic",
             TlsModel::InitialExec => "initial-exec",
             TlsModel::LocalExec => "local-exec",
+            TlsModel::Emulated => "emulated",
         }
         .to_json()
     }
@@ -2190,9 +2193,6 @@ pub struct TargetOptions {
 
     /// Whether the target supports XRay instrumentation.
     pub supports_xray: bool,
-
-    /// Forces the use of emulated TLS (__emutls_get_address)
-    pub force_emulated_tls: bool,
 }
 
 /// Add arguments for the given flavor and also for its "twin" flavors
@@ -2408,7 +2408,6 @@ impl Default for TargetOptions {
             entry_name: "main".into(),
             entry_abi: Conv::C,
             supports_xray: false,
-            force_emulated_tls: false,
         }
     }
 }
@@ -3112,7 +3111,6 @@ impl Target {
         key!(entry_name);
         key!(entry_abi, Conv)?;
         key!(supports_xray, bool);
-        key!(force_emulated_tls, bool);
 
         if base.is_builtin {
             // This can cause unfortunate ICEs later down the line.
@@ -3368,7 +3366,6 @@ impl ToJson for Target {
         target_option_val!(entry_name);
         target_option_val!(entry_abi);
         target_option_val!(supports_xray);
-        target_option_val!(force_emulated_tls);
 
         if let Some(abi) = self.default_adjusted_cabi {
             d.insert("default-adjusted-cabi".into(), Abi::name(abi).to_json());
diff --git a/library/std/src/sys/unix/thread_local_dtor.rs b/library/std/src/sys/unix/thread_local_dtor.rs
index 667fd516962..ac85531c372 100644
--- a/library/std/src/sys/unix/thread_local_dtor.rs
+++ b/library/std/src/sys/unix/thread_local_dtor.rs
@@ -12,7 +12,13 @@
 // compiling from a newer linux to an older linux, so we also have a
 // fallback implementation to use as well.
 #[allow(unexpected_cfgs)]
-#[cfg(any(target_os = "linux", target_os = "fuchsia", target_os = "redox", target_os = "hurd"))]
+#[cfg(any(
+    target_os = "linux",
+    target_os = "android",
+    target_os = "fuchsia",
+    target_os = "redox",
+    target_os = "hurd"
+))]
 // FIXME: The Rust compiler currently omits weakly function definitions (i.e.,
 // __cxa_thread_atexit_impl) and its metadata from LLVM IR.
 #[no_sanitize(cfi, kcfi)]
diff --git a/src/doc/unstable-book/src/compiler-flags/tls-model.md b/src/doc/unstable-book/src/compiler-flags/tls-model.md
index 8b19e785c6a..b7024ba88de 100644
--- a/src/doc/unstable-book/src/compiler-flags/tls-model.md
+++ b/src/doc/unstable-book/src/compiler-flags/tls-model.md
@@ -20,6 +20,8 @@ loaded at program startup.
 The TLS data must not be in a library loaded after startup (via `dlopen`).
 - `local-exec` - model usable only if the TLS data is defined directly in the executable,
 but not in a shared library, and is accessed only from that executable.
+- `emulated` - Uses thread-specific data keys to implement emulated TLS.
+It is like using a general-dynamic TLS model for all modes.
 
 `rustc` and LLVM may use a more optimized model than specified if they know that we are producing
 an executable rather than a library, or that the `static` item is private enough.
-- 
cgit 1.4.1-3-g733a5


From 856b55fb34050a56f4ce2a5e171d8a7dca19729c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 6 Dec 2023 20:36:44 +1100
Subject: De-pub some functions.

---
 compiler/rustc_lexer/src/unescape.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 717b042fbda..d98a702427e 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -167,7 +167,7 @@ impl Mode {
     }
 
     /// Non-byte literals should have `\xXX` escapes that are within the ASCII range.
-    pub fn ascii_escapes_should_be_ascii(self) -> bool {
+    fn ascii_escapes_should_be_ascii(self) -> bool {
         match self {
             Mode::Char | Mode::Str | Mode::RawStr => true,
             Mode::Byte | Mode::ByteStr | Mode::RawByteStr | Mode::CStr | Mode::RawCStr => false,
@@ -175,7 +175,7 @@ impl Mode {
     }
 
     /// Whether characters within the literal must be within the ASCII range
-    pub fn characters_should_be_ascii(self) -> bool {
+    fn characters_should_be_ascii(self) -> bool {
         match self {
             Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
             Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false,
@@ -183,7 +183,7 @@ impl Mode {
     }
 
     /// Byte literals do not allow unicode escape.
-    pub fn is_unicode_escape_disallowed(self) -> bool {
+    fn is_unicode_escape_disallowed(self) -> bool {
         match self {
             Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
             Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false,
-- 
cgit 1.4.1-3-g733a5


From e290582abff724811eca6dd021f97955cb13a3b8 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 6 Dec 2023 20:37:02 +1100
Subject: Identify impossible cases in `ascii_escapes_should_be_ascii`.

Raw strings (of all kinds) don't support escapes, so this function
should never be called on them.
---
 compiler/rustc_lexer/src/unescape.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index d98a702427e..ddbe826f570 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -169,8 +169,9 @@ impl Mode {
     /// Non-byte literals should have `\xXX` escapes that are within the ASCII range.
     fn ascii_escapes_should_be_ascii(self) -> bool {
         match self {
-            Mode::Char | Mode::Str | Mode::RawStr => true,
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr | Mode::CStr | Mode::RawCStr => false,
+            Mode::Char | Mode::Str => true,
+            Mode::Byte | Mode::ByteStr | Mode::CStr => false,
+            Mode::RawStr | Mode::RawByteStr | Mode::RawCStr => unreachable!(),
         }
     }
 
-- 
cgit 1.4.1-3-g733a5


From c6bbb376a24c1397ca13078192418ef746382b10 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 6 Dec 2023 20:39:08 +1100
Subject: Fix an out-of-date comment.

---
 compiler/rustc_lexer/src/unescape.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index ddbe826f570..dab656b35f9 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -395,7 +395,7 @@ where
     let mut chars = src.chars();
 
     // The `start` and `end` computation here matches the one in
-    // `unescape_str_or_byte_str` for consistency, even though this function
+    // `unescape_str_common` for consistency, even though this function
     // doesn't have to worry about skipping any chars.
     while let Some(c) = chars.next() {
         let start = src.len() - chars.as_str().len() - c.len_utf8();
-- 
cgit 1.4.1-3-g733a5


From 47e6e5ee675a78cd8f6f11daa0611f577f6e2f7b Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 7 Dec 2023 11:12:48 +1100
Subject: coverage: Avoid unnecessary macros in unit tests

These macros don't provide enough value to justify their complexity, when they
can just as easily be functions instead.
---
 Cargo.lock                                         |  5 --
 compiler/rustc_mir_transform/Cargo.toml            |  5 --
 .../src/coverage/test_macros/Cargo.toml            |  7 --
 .../src/coverage/test_macros/src/lib.rs            |  6 --
 compiler/rustc_mir_transform/src/coverage/tests.rs | 90 ++++++++--------------
 5 files changed, 33 insertions(+), 80 deletions(-)
 delete mode 100644 compiler/rustc_mir_transform/src/coverage/test_macros/Cargo.toml
 delete mode 100644 compiler/rustc_mir_transform/src/coverage/test_macros/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index 7f627e2ce6e..1d7f93c789a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -795,10 +795,6 @@ dependencies = [
  "rustc-demangle",
 ]
 
-[[package]]
-name = "coverage_test_macros"
-version = "0.0.0"
-
 [[package]]
 name = "cpufeatures"
 version = "0.2.8"
@@ -4266,7 +4262,6 @@ dependencies = [
 name = "rustc_mir_transform"
 version = "0.0.0"
 dependencies = [
- "coverage_test_macros",
  "either",
  "itertools",
  "rustc_arena",
diff --git a/compiler/rustc_mir_transform/Cargo.toml b/compiler/rustc_mir_transform/Cargo.toml
index c2ca0a6bcb8..9cc083edb44 100644
--- a/compiler/rustc_mir_transform/Cargo.toml
+++ b/compiler/rustc_mir_transform/Cargo.toml
@@ -27,8 +27,3 @@ rustc_trait_selection = { path = "../rustc_trait_selection" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"
 # tidy-alphabetical-end
-
-[dev-dependencies]
-# tidy-alphabetical-start
-coverage_test_macros = { path = "src/coverage/test_macros" }
-# tidy-alphabetical-end
diff --git a/compiler/rustc_mir_transform/src/coverage/test_macros/Cargo.toml b/compiler/rustc_mir_transform/src/coverage/test_macros/Cargo.toml
deleted file mode 100644
index f753caa9124..00000000000
--- a/compiler/rustc_mir_transform/src/coverage/test_macros/Cargo.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-[package]
-name = "coverage_test_macros"
-version = "0.0.0"
-edition = "2021"
-
-[lib]
-proc-macro = true
diff --git a/compiler/rustc_mir_transform/src/coverage/test_macros/src/lib.rs b/compiler/rustc_mir_transform/src/coverage/test_macros/src/lib.rs
deleted file mode 100644
index f41adf667ec..00000000000
--- a/compiler/rustc_mir_transform/src/coverage/test_macros/src/lib.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-use proc_macro::TokenStream;
-
-#[proc_macro]
-pub fn let_bcb(item: TokenStream) -> TokenStream {
-    format!("let bcb{item} = graph::BasicCoverageBlock::from_usize({item});").parse().unwrap()
-}
diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs
index 702fe5f563e..302cbf05d78 100644
--- a/compiler/rustc_mir_transform/src/coverage/tests.rs
+++ b/compiler/rustc_mir_transform/src/coverage/tests.rs
@@ -27,8 +27,6 @@
 use super::counters;
 use super::graph::{self, BasicCoverageBlock};
 
-use coverage_test_macros::let_bcb;
-
 use itertools::Itertools;
 use rustc_data_structures::graph::WithNumNodes;
 use rustc_data_structures::graph::WithSuccessors;
@@ -37,6 +35,10 @@ use rustc_middle::mir::*;
 use rustc_middle::ty;
 use rustc_span::{self, BytePos, Pos, Span, DUMMY_SP};
 
+fn bcb(index: u32) -> BasicCoverageBlock {
+    BasicCoverageBlock::from_u32(index)
+}
+
 // All `TEMP_BLOCK` targets should be replaced before calling `to_body() -> mir::Body`.
 const TEMP_BLOCK: BasicBlock = BasicBlock::MAX;
 
@@ -300,12 +302,15 @@ fn goto_switchint<'a>() -> Body<'a> {
     mir_body
 }
 
-macro_rules! assert_successors {
-    ($basic_coverage_blocks:ident, $i:ident, [$($successor:ident),*]) => {
-        let mut successors = $basic_coverage_blocks.successors[$i].clone();
-        successors.sort_unstable();
-        assert_eq!(successors, vec![$($successor),*]);
-    }
+#[track_caller]
+fn assert_successors(
+    basic_coverage_blocks: &graph::CoverageGraph,
+    bcb: BasicCoverageBlock,
+    expected_successors: &[BasicCoverageBlock],
+) {
+    let mut successors = basic_coverage_blocks.successors[bcb].clone();
+    successors.sort_unstable();
+    assert_eq!(successors, expected_successors);
 }
 
 #[test]
@@ -334,13 +339,9 @@ fn test_covgraph_goto_switchint() {
         basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>()
     );
 
-    let_bcb!(0);
-    let_bcb!(1);
-    let_bcb!(2);
-
-    assert_successors!(basic_coverage_blocks, bcb0, [bcb1, bcb2]);
-    assert_successors!(basic_coverage_blocks, bcb1, []);
-    assert_successors!(basic_coverage_blocks, bcb2, []);
+    assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1), bcb(2)]);
+    assert_successors(&basic_coverage_blocks, bcb(1), &[]);
+    assert_successors(&basic_coverage_blocks, bcb(2), &[]);
 }
 
 /// Create a mock `Body` with a loop.
@@ -418,15 +419,10 @@ fn test_covgraph_switchint_then_loop_else_return() {
         basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>()
     );
 
-    let_bcb!(0);
-    let_bcb!(1);
-    let_bcb!(2);
-    let_bcb!(3);
-
-    assert_successors!(basic_coverage_blocks, bcb0, [bcb1]);
-    assert_successors!(basic_coverage_blocks, bcb1, [bcb2, bcb3]);
-    assert_successors!(basic_coverage_blocks, bcb2, []);
-    assert_successors!(basic_coverage_blocks, bcb3, [bcb1]);
+    assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1)]);
+    assert_successors(&basic_coverage_blocks, bcb(1), &[bcb(2), bcb(3)]);
+    assert_successors(&basic_coverage_blocks, bcb(2), &[]);
+    assert_successors(&basic_coverage_blocks, bcb(3), &[bcb(1)]);
 }
 
 /// Create a mock `Body` with nested loops.
@@ -546,21 +542,13 @@ fn test_covgraph_switchint_loop_then_inner_loop_else_break() {
         basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>()
     );
 
-    let_bcb!(0);
-    let_bcb!(1);
-    let_bcb!(2);
-    let_bcb!(3);
-    let_bcb!(4);
-    let_bcb!(5);
-    let_bcb!(6);
-
-    assert_successors!(basic_coverage_blocks, bcb0, [bcb1]);
-    assert_successors!(basic_coverage_blocks, bcb1, [bcb2, bcb3]);
-    assert_successors!(basic_coverage_blocks, bcb2, []);
-    assert_successors!(basic_coverage_blocks, bcb3, [bcb4]);
-    assert_successors!(basic_coverage_blocks, bcb4, [bcb5, bcb6]);
-    assert_successors!(basic_coverage_blocks, bcb5, [bcb1]);
-    assert_successors!(basic_coverage_blocks, bcb6, [bcb4]);
+    assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1)]);
+    assert_successors(&basic_coverage_blocks, bcb(1), &[bcb(2), bcb(3)]);
+    assert_successors(&basic_coverage_blocks, bcb(2), &[]);
+    assert_successors(&basic_coverage_blocks, bcb(3), &[bcb(4)]);
+    assert_successors(&basic_coverage_blocks, bcb(4), &[bcb(5), bcb(6)]);
+    assert_successors(&basic_coverage_blocks, bcb(5), &[bcb(1)]);
+    assert_successors(&basic_coverage_blocks, bcb(6), &[bcb(4)]);
 }
 
 #[test]
@@ -595,10 +583,7 @@ fn test_find_loop_backedges_one() {
         backedges
     );
 
-    let_bcb!(1);
-    let_bcb!(3);
-
-    assert_eq!(backedges[bcb1], vec![bcb3]);
+    assert_eq!(backedges[bcb(1)], &[bcb(3)]);
 }
 
 #[test]
@@ -613,13 +598,8 @@ fn test_find_loop_backedges_two() {
         backedges
     );
 
-    let_bcb!(1);
-    let_bcb!(4);
-    let_bcb!(5);
-    let_bcb!(6);
-
-    assert_eq!(backedges[bcb1], vec![bcb5]);
-    assert_eq!(backedges[bcb4], vec![bcb6]);
+    assert_eq!(backedges[bcb(1)], &[bcb(5)]);
+    assert_eq!(backedges[bcb(4)], &[bcb(6)]);
 }
 
 #[test]
@@ -632,13 +612,11 @@ fn test_traverse_coverage_with_loops() {
         traversed_in_order.push(bcb);
     }
 
-    let_bcb!(6);
-
     // bcb0 is visited first. Then bcb1 starts the first loop, and all remaining nodes, *except*
     // bcb6 are inside the first loop.
     assert_eq!(
         *traversed_in_order.last().expect("should have elements"),
-        bcb6,
+        bcb(6),
         "bcb6 should not be visited until all nodes inside the first loop have been visited"
     );
 }
@@ -656,20 +634,18 @@ fn test_make_bcb_counters() {
         coverage_counters.make_bcb_counters(&basic_coverage_blocks, bcb_has_coverage_spans);
         assert_eq!(coverage_counters.num_expressions(), 0);
 
-        let_bcb!(1);
         assert_eq!(
             0, // bcb1 has a `Counter` with id = 0
-            match coverage_counters.bcb_counter(bcb1).expect("should have a counter") {
+            match coverage_counters.bcb_counter(bcb(1)).expect("should have a counter") {
                 counters::BcbCounter::Counter { id, .. } => id,
                 _ => panic!("expected a Counter"),
             }
             .as_u32()
         );
 
-        let_bcb!(2);
         assert_eq!(
             1, // bcb2 has a `Counter` with id = 1
-            match coverage_counters.bcb_counter(bcb2).expect("should have a counter") {
+            match coverage_counters.bcb_counter(bcb(2)).expect("should have a counter") {
                 counters::BcbCounter::Counter { id, .. } => id,
                 _ => panic!("expected a Counter"),
             }
-- 
cgit 1.4.1-3-g733a5


From aa58ccb53e289aa465a070cc27d2d773d274d1d1 Mon Sep 17 00:00:00 2001
From: Ben Kimock <kimockb@gmail.com>
Date: Wed, 6 Dec 2023 21:18:37 -0500
Subject: Tell MirUsedCollector that the pointer alignment checks calls its
 panic symbol

---
 compiler/rustc_monomorphize/src/collector.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index b882a038711..8609aa2bfb5 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -844,6 +844,9 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
             mir::TerminatorKind::Assert { ref msg, .. } => {
                 let lang_item = match &**msg {
                     mir::AssertKind::BoundsCheck { .. } => LangItem::PanicBoundsCheck,
+                    mir::AssertKind::MisalignedPointerDereference { .. } => {
+                        LangItem::PanicMisalignedPointerDereference
+                    }
                     _ => LangItem::Panic,
                 };
                 push_mono_lang_item(self, lang_item);
-- 
cgit 1.4.1-3-g733a5


From ec0110be0961846b1c69d7208f07c740564e0d8a Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 7 Dec 2023 14:04:10 +1100
Subject: coverage: Merge refined spans in a separate final pass

This makes `push_refined_span` trivial, which will let us inline it and benefit
from partial borrows of `refined_spans`.
---
 compiler/rustc_mir_transform/src/coverage/spans.rs | 29 ++++++++++++----------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index 4db0a1db166..cce2910244b 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -89,10 +89,10 @@ impl CoverageSpan {
         }
     }
 
-    pub fn merge_from(&mut self, mut other: CoverageSpan) {
-        debug_assert!(self.is_mergeable(&other));
+    pub fn merge_from(&mut self, other: &Self) {
+        debug_assert!(self.is_mergeable(other));
         self.span = self.span.to(other.span);
-        self.merged_spans.append(&mut other.merged_spans);
+        self.merged_spans.extend_from_slice(&other.merged_spans);
     }
 
     pub fn cutoff_statements_at(&mut self, cutoff_pos: BytePos) {
@@ -267,7 +267,7 @@ impl<'a> CoverageSpansGenerator<'a> {
             if curr.is_mergeable(prev) {
                 debug!("  same bcb (and neither is a closure), merge with prev={prev:?}");
                 let prev = self.take_prev();
-                self.curr_mut().merge_from(prev);
+                self.curr_mut().merge_from(&prev);
                 self.maybe_push_macro_name_span();
             // Note that curr.span may now differ from curr_original_span
             } else if prev.span.hi() <= curr.span.lo() {
@@ -346,6 +346,17 @@ impl<'a> CoverageSpansGenerator<'a> {
             self.push_refined_span(prev);
         }
 
+        // Do one last merge pass, to simplify the output.
+        self.refined_spans.dedup_by(|b, a| {
+            if a.is_mergeable(b) {
+                debug!(?a, ?b, "merging list-adjacent refined spans");
+                a.merge_from(b);
+                true
+            } else {
+                false
+            }
+        });
+
         // Remove `CoverageSpan`s derived from closures, originally added to ensure the coverage
         // regions for the current function leave room for the closure's own coverage regions
         // (injected separately, from the closure's own MIR).
@@ -354,15 +365,7 @@ impl<'a> CoverageSpansGenerator<'a> {
     }
 
     fn push_refined_span(&mut self, covspan: CoverageSpan) {
-        if let Some(last) = self.refined_spans.last_mut()
-            && last.is_mergeable(&covspan)
-        {
-            // Instead of pushing the new span, merge it with the last refined span.
-            debug!(?last, ?covspan, "merging new refined span with last refined span");
-            last.merge_from(covspan);
-        } else {
-            self.refined_spans.push(covspan);
-        }
+        self.refined_spans.push(covspan);
     }
 
     /// If `curr` is part of a new macro expansion, carve out and push a separate
-- 
cgit 1.4.1-3-g733a5


From 9089d287801f93150845202066ac1a093f343e1a Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 7 Dec 2023 16:08:46 +1100
Subject: coverage: Inline `push_refined_span`

---
 compiler/rustc_mir_transform/src/coverage/spans.rs | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index cce2910244b..df26ff36998 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -275,7 +275,7 @@ impl<'a> CoverageSpansGenerator<'a> {
                     "  different bcbs and disjoint spans, so keep curr for next iter, and add prev={prev:?}",
                 );
                 let prev = self.take_prev();
-                self.push_refined_span(prev);
+                self.refined_spans.push(prev);
                 self.maybe_push_macro_name_span();
             } else if prev.is_closure {
                 // drop any equal or overlapping span (`curr`) and keep `prev` to test again in the
@@ -326,7 +326,7 @@ impl<'a> CoverageSpansGenerator<'a> {
         // It is never used as a field after this point.
         for dup in std::mem::take(&mut self.pending_dups) {
             debug!("    ...adding at least one pending dup={:?}", dup);
-            self.push_refined_span(dup);
+            self.refined_spans.push(dup);
         }
 
         // Async functions wrap a closure that implements the body to be executed. The enclosing
@@ -343,7 +343,7 @@ impl<'a> CoverageSpansGenerator<'a> {
         };
 
         if !body_ends_with_closure {
-            self.push_refined_span(prev);
+            self.refined_spans.push(prev);
         }
 
         // Do one last merge pass, to simplify the output.
@@ -364,10 +364,6 @@ impl<'a> CoverageSpansGenerator<'a> {
         self.refined_spans
     }
 
-    fn push_refined_span(&mut self, covspan: CoverageSpan) {
-        self.refined_spans.push(covspan);
-    }
-
     /// If `curr` is part of a new macro expansion, carve out and push a separate
     /// span that ends just after the macro name and its subsequent `!`.
     fn maybe_push_macro_name_span(&mut self) {
@@ -400,7 +396,7 @@ impl<'a> CoverageSpansGenerator<'a> {
             "  and curr starts a new macro expansion, so add a new span just for \
             the macro `{visible_macro}!`, new span={macro_name_cov:?}",
         );
-        self.push_refined_span(macro_name_cov);
+        self.refined_spans.push(macro_name_cov);
     }
 
     fn curr(&self) -> &CoverageSpan {
@@ -462,7 +458,7 @@ impl<'a> CoverageSpansGenerator<'a> {
             let mut pending_dups = std::mem::take(&mut self.pending_dups);
             for dup in pending_dups.drain(..) {
                 debug!("    ...adding at least one pending={:?}", dup);
-                self.push_refined_span(dup);
+                self.refined_spans.push(dup);
             }
             // The list of dups is now empty, but we can recycle its capacity.
             assert!(pending_dups.is_empty() && self.pending_dups.is_empty());
@@ -528,10 +524,10 @@ impl<'a> CoverageSpansGenerator<'a> {
                 for mut dup in pending_dups.iter().cloned() {
                     dup.span = dup.span.with_hi(left_cutoff);
                     debug!("    ...and at least one pre_closure dup={:?}", dup);
-                    self.push_refined_span(dup);
+                    self.refined_spans.push(dup);
                 }
             }
-            self.push_refined_span(pre_closure);
+            self.refined_spans.push(pre_closure);
         }
 
         if has_post_closure_span {
@@ -545,7 +541,7 @@ impl<'a> CoverageSpansGenerator<'a> {
                 dup.span = dup.span.with_lo(right_cutoff);
             }
             let closure_covspan = self.take_curr(); // Prevent this curr from becoming prev.
-            self.push_refined_span(closure_covspan); // since self.prev() was already updated
+            self.refined_spans.push(closure_covspan); // since self.prev() was already updated
         } else {
             pending_dups.clear();
         }
@@ -648,7 +644,7 @@ impl<'a> CoverageSpansGenerator<'a> {
             } else {
                 debug!("  ... adding modified prev={:?}", self.prev());
                 let prev = self.take_prev();
-                self.push_refined_span(prev);
+                self.refined_spans.push(prev);
             }
         } else {
             // with `pending_dups`, `prev` cannot have any statements that don't overlap
-- 
cgit 1.4.1-3-g733a5


From 9a4321518c3ed461c9033bf02dcc1fa66da96ee3 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 7 Dec 2023 14:07:25 +1100
Subject: coverage: Simplify code that pushes to `refined_spans`

---
 compiler/rustc_mir_transform/src/coverage/spans.rs | 40 ++++++++--------------
 1 file changed, 14 insertions(+), 26 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index df26ff36998..c415a832994 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -322,9 +322,8 @@ impl<'a> CoverageSpansGenerator<'a> {
         let prev = self.take_prev();
         debug!("    AT END, adding last prev={prev:?}");
 
-        // Take `pending_dups` so that we can drain it while calling self methods.
-        // It is never used as a field after this point.
-        for dup in std::mem::take(&mut self.pending_dups) {
+        // Drain any remaining dups into the output.
+        for dup in self.pending_dups.drain(..) {
             debug!("    ...adding at least one pending dup={:?}", dup);
             self.refined_spans.push(dup);
         }
@@ -453,19 +452,14 @@ impl<'a> CoverageSpansGenerator<'a> {
             previous iteration, or prev started a new disjoint span"
         );
         if last_dup.span.hi() <= self.curr().span.lo() {
-            // Temporarily steal `pending_dups` into a local, so that we can
-            // drain it while calling other self methods.
-            let mut pending_dups = std::mem::take(&mut self.pending_dups);
-            for dup in pending_dups.drain(..) {
+            for dup in self.pending_dups.drain(..) {
                 debug!("    ...adding at least one pending={:?}", dup);
                 self.refined_spans.push(dup);
             }
-            // The list of dups is now empty, but we can recycle its capacity.
-            assert!(pending_dups.is_empty() && self.pending_dups.is_empty());
-            self.pending_dups = pending_dups;
         } else {
             self.pending_dups.clear();
         }
+        assert!(self.pending_dups.is_empty());
     }
 
     /// Advance `prev` to `curr` (if any), and `curr` to the next `CoverageSpan` in sorted order.
@@ -512,21 +506,17 @@ impl<'a> CoverageSpansGenerator<'a> {
         let has_pre_closure_span = prev.span.lo() < right_cutoff;
         let has_post_closure_span = prev.span.hi() > right_cutoff;
 
-        // Temporarily steal `pending_dups` into a local, so that we can
-        // mutate and/or drain it while calling other self methods.
-        let mut pending_dups = std::mem::take(&mut self.pending_dups);
-
         if has_pre_closure_span {
             let mut pre_closure = self.prev().clone();
             pre_closure.span = pre_closure.span.with_hi(left_cutoff);
             debug!("  prev overlaps a closure. Adding span for pre_closure={:?}", pre_closure);
-            if !pending_dups.is_empty() {
-                for mut dup in pending_dups.iter().cloned() {
-                    dup.span = dup.span.with_hi(left_cutoff);
-                    debug!("    ...and at least one pre_closure dup={:?}", dup);
-                    self.refined_spans.push(dup);
-                }
+
+            for mut dup in self.pending_dups.iter().cloned() {
+                dup.span = dup.span.with_hi(left_cutoff);
+                debug!("    ...and at least one pre_closure dup={:?}", dup);
+                self.refined_spans.push(dup);
             }
+
             self.refined_spans.push(pre_closure);
         }
 
@@ -536,19 +526,17 @@ impl<'a> CoverageSpansGenerator<'a> {
             // about how the `CoverageSpan`s are ordered.)
             self.prev_mut().span = self.prev().span.with_lo(right_cutoff);
             debug!("  Mutated prev.span to start after the closure. prev={:?}", self.prev());
-            for dup in pending_dups.iter_mut() {
+
+            for dup in &mut self.pending_dups {
                 debug!("    ...and at least one overlapping dup={:?}", dup);
                 dup.span = dup.span.with_lo(right_cutoff);
             }
+
             let closure_covspan = self.take_curr(); // Prevent this curr from becoming prev.
             self.refined_spans.push(closure_covspan); // since self.prev() was already updated
         } else {
-            pending_dups.clear();
+            self.pending_dups.clear();
         }
-
-        // Restore the modified post-closure spans, or the empty vector's capacity.
-        assert!(self.pending_dups.is_empty());
-        self.pending_dups = pending_dups;
     }
 
     /// Called if `curr.span` equals `prev_original_span` (and potentially equal to all
-- 
cgit 1.4.1-3-g733a5


From bf681dcfb52c9f6a54ab4e72a8f827f702fcd1a7 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Wed, 6 Dec 2023 15:33:01 +0100
Subject: Fix display of features in rustdoc

---
 src/librustdoc/html/static/css/rustdoc.css | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/librustdoc/html/static/css/rustdoc.css b/src/librustdoc/html/static/css/rustdoc.css
index b898eb5d381..6e61969a8c1 100644
--- a/src/librustdoc/html/static/css/rustdoc.css
+++ b/src/librustdoc/html/static/css/rustdoc.css
@@ -1081,15 +1081,9 @@ so that we can apply CSS-filters to change the arrow color in themes */
 }
 
 .item-info .stab {
-	/* This min-height is needed to unify the height of the stab elements because some of them
-	   have emojis.
-	*/
-	min-height: 36px;
-	display: flex;
+	display: block;
 	padding: 3px;
 	margin-bottom: 5px;
-	align-items: center;
-	vertical-align: text-bottom;
 }
 .item-name .stab {
 	margin-left: 0.3125em;
@@ -1112,17 +1106,26 @@ so that we can apply CSS-filters to change the arrow color in themes */
 	color: var(--stab-code-color);
 }
 
-.stab .emoji {
+.stab .emoji, .item-info .stab::before {
 	font-size: 1.25rem;
+}
+.stab .emoji {
 	margin-right: 0.3rem;
 }
+.item-info .stab::before {
+	/* ensure badges with emoji and without it have same height */
+	content: "\0";
+	width: 0;
+	display: inline-block;
+	color: transparent;
+}
 
 /* Black one-pixel outline around emoji shapes */
 .emoji {
 	text-shadow:
 		1px 0 0 black,
 		-1px 0 0 black,
-		0  1px 0 black,
+		0 1px 0 black,
 		0 -1px 0 black;
 }
 
-- 
cgit 1.4.1-3-g733a5


From 9e1797baf1639156866b6e32da5ed547e07e4053 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Wed, 6 Dec 2023 15:42:02 +0100
Subject: Extend GUI tests for `doc_cfg`

---
 tests/rustdoc-gui/item-info.goml      | 15 +++++++++++++++
 tests/rustdoc-gui/src/lib2/Cargo.toml |  7 +++++++
 tests/rustdoc-gui/src/lib2/lib.rs     |  9 +++++++++
 3 files changed, 31 insertions(+)

diff --git a/tests/rustdoc-gui/item-info.goml b/tests/rustdoc-gui/item-info.goml
index 030ff8f8a3e..b46d4255ee5 100644
--- a/tests/rustdoc-gui/item-info.goml
+++ b/tests/rustdoc-gui/item-info.goml
@@ -8,7 +8,22 @@ assert-size: (".item-info", {"width": 840})
 assert-size: (".item-info .stab", {"width": 289})
 assert-position: (".item-info .stab", {"x": 245})
 
+// We check that the display of the feature elements is not broken. It serves as regression
+// test for <https://github.com/rust-lang/rust/issues/118615>.
+set-window-size: (850, 800)
+store-position: (
+    "//*[@class='stab portability']//code[text()='Win32_System']",
+    {"x": first_line_x, "y": first_line_y},
+)
+store-position: (
+    "//*[@class='stab portability']//code[text()='Win32_System_Diagnostics']",
+    {"x": second_line_x, "y": second_line_y},
+)
+assert: |first_line_x| != |second_line_x| && |first_line_x| == 516 && |second_line_x| == 272
+assert: |first_line_y| != |second_line_y| && |first_line_y| == 688 && |second_line_y| == 711
+
 // Now we ensure that they're not rendered on the same line.
+set-window-size: (1100, 800)
 go-to: "file://" + |DOC_PATH| + "/lib2/trait.Trait.html"
 // We first ensure that there are two item info on the trait.
 assert-count: ("#main-content > .item-info .stab", 2)
diff --git a/tests/rustdoc-gui/src/lib2/Cargo.toml b/tests/rustdoc-gui/src/lib2/Cargo.toml
index 8bca77ff834..6c4ca27d550 100644
--- a/tests/rustdoc-gui/src/lib2/Cargo.toml
+++ b/tests/rustdoc-gui/src/lib2/Cargo.toml
@@ -6,6 +6,13 @@ edition = "2018"
 [lib]
 path = "lib.rs"
 
+[features]
+Win32 = ["Win32_System"]
+Win32_System = ["Win32_System_Diagnostics"]
+Win32_System_Diagnostics = ["Win32_System_Diagnostics_Debug"]
+Win32_System_Diagnostics_Debug = []
+default = ["Win32"]
+
 [dependencies]
 implementors = { path = "./implementors" }
 http = { path = "./http" }
diff --git a/tests/rustdoc-gui/src/lib2/lib.rs b/tests/rustdoc-gui/src/lib2/lib.rs
index a2a3c31878b..b467b044052 100644
--- a/tests/rustdoc-gui/src/lib2/lib.rs
+++ b/tests/rustdoc-gui/src/lib2/lib.rs
@@ -1,6 +1,7 @@
 // ignore-tidy-linelength
 
 #![feature(doc_cfg)]
+#![feature(doc_auto_cfg)]
 
 pub mod another_folder;
 pub mod another_mod;
@@ -28,6 +29,14 @@ impl Foo {
     /// Some documentation
     /// # A Heading
     pub fn a_method(&self) {}
+
+    #[cfg(all(
+        feature = "Win32",
+        feature = "Win32_System",
+        feature = "Win32_System_Diagnostics",
+        feature = "Win32_System_Diagnostics_Debug"
+    ))]
+    pub fn lot_of_features() {}
 }
 
 #[doc(cfg(feature = "foo-method"))]
-- 
cgit 1.4.1-3-g733a5


From 40aa9f4fd914cbbe043da1d5ffdecae287354dce Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Thu, 7 Dec 2023 16:13:09 +0100
Subject: avoid instantiating infer vars with infer

---
 compiler/rustc_infer/src/infer/combine.rs       | 8 +++++++-
 compiler/rustc_infer/src/infer/type_variable.rs | 5 ++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs
index 759ebaa1d1e..dfae279324f 100644
--- a/compiler/rustc_infer/src/infer/combine.rs
+++ b/compiler/rustc_infer/src/infer/combine.rs
@@ -34,6 +34,7 @@ use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue, EffectVa
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::relate::{RelateResult, TypeRelation};
+use rustc_middle::ty::TyVar;
 use rustc_middle::ty::{self, InferConst, ToPredicate, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::ty::{IntType, UintType};
 use rustc_span::DUMMY_SP;
@@ -459,7 +460,12 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
             ambient_variance,
         )?;
 
-        self.infcx.inner.borrow_mut().type_variables().instantiate(b_vid, b_ty);
+        // Constrain `b_vid` to the generalized type `b_ty`.
+        if let &ty::Infer(TyVar(b_ty_vid)) = b_ty.kind() {
+            self.infcx.inner.borrow_mut().type_variables().equate(b_vid, b_ty_vid);
+        } else {
+            self.infcx.inner.borrow_mut().type_variables().instantiate(b_vid, b_ty);
+        }
 
         if needs_wf {
             self.obligations.push(Obligation::new(
diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs
index bc83f8d3f96..bd6f905c824 100644
--- a/compiler/rustc_infer/src/infer/type_variable.rs
+++ b/compiler/rustc_infer/src/infer/type_variable.rs
@@ -229,12 +229,11 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
     /// Precondition: `vid` must not have been previously instantiated.
     pub fn instantiate(&mut self, vid: ty::TyVid, ty: Ty<'tcx>) {
         let vid = self.root_var(vid);
+        debug_assert!(!ty.is_ty_var(), "instantiating ty var with var: {vid:?} {ty:?}");
         debug_assert!(self.probe(vid).is_unknown());
         debug_assert!(
             self.eq_relations().probe_value(vid).is_unknown(),
-            "instantiating type variable `{:?}` twice: new-value = {:?}, old-value={:?}",
-            vid,
-            ty,
+            "instantiating type variable `{vid:?}` twice: new-value = {ty:?}, old-value={:?}",
             self.eq_relations().probe_value(vid)
         );
         self.eq_relations().union_value(vid, TypeVariableValue::Known { value: ty });
-- 
cgit 1.4.1-3-g733a5


From c3bb1b50a5c1b354805f65170fb8034a86cc9598 Mon Sep 17 00:00:00 2001
From: Chris Eckhardt <29362105+CLEckhardt@users.noreply.github.com>
Date: Fri, 1 Dec 2023 10:30:09 -0600
Subject: Elaborate on ip_addr bit conversion endianness

Adds explanation of how endianness is handled when converting `Ipv4Addr`
and `Ipv6Addr` to and from bits.

Addresses #113744
---
 library/core/src/net/ip_addr.rs | 52 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 48 insertions(+), 4 deletions(-)

diff --git a/library/core/src/net/ip_addr.rs b/library/core/src/net/ip_addr.rs
index 77f85215d71..dafc58d3a7d 100644
--- a/library/core/src/net/ip_addr.rs
+++ b/library/core/src/net/ip_addr.rs
@@ -468,7 +468,13 @@ impl Ipv4Addr {
     #[unstable(feature = "ip_bits", issue = "113744")]
     pub const BITS: u32 = 32;
 
-    /// Converts an IPv4 address into host byte order `u32`.
+    /// Converts an IPv4 address into a `u32` representation using native byte order.
+    ///
+    /// Although IPv4 addresses are big-endian, the `u32` value will use the target platform's
+    /// native byte order. That is, the `u32` value is an integer representation of the IPv4
+    /// address and not an integer interpretation of the IPv4 address's big-endian bitstring. This
+    /// means that the `u32` value masked with `0xffffff00` will set the last octet in the address
+    /// to 0, regardless of the target platform's endianness.
     ///
     /// # Examples
     ///
@@ -479,6 +485,16 @@ impl Ipv4Addr {
     /// let addr = Ipv4Addr::new(0x12, 0x34, 0x56, 0x78);
     /// assert_eq!(0x12345678, addr.to_bits());
     /// ```
+    ///
+    /// ```
+    /// #![feature(ip_bits)]
+    /// use std::net::Ipv4Addr;
+    ///
+    /// let addr = Ipv4Addr::new(0x12, 0x34, 0x56, 0x78);
+    /// let addr_bits = addr.to_bits() & 0xffffff00;
+    /// assert_eq!(Ipv4Addr::new(0x12, 0x34, 0x56, 0x00), Ipv4Addr::from_bits(addr_bits));
+    ///
+    /// ```
     #[rustc_const_unstable(feature = "ip_bits", issue = "113744")]
     #[unstable(feature = "ip_bits", issue = "113744")]
     #[must_use]
@@ -487,7 +503,9 @@ impl Ipv4Addr {
         u32::from_be_bytes(self.octets)
     }
 
-    /// Converts a host byte order `u32` into an IPv4 address.
+    /// Converts a native byte order `u32` into an IPv4 address.
+    ///
+    /// See [`Ipv4Addr::to_bits`] for an explanation on endianness.
     ///
     /// # Examples
     ///
@@ -1224,7 +1242,13 @@ impl Ipv6Addr {
     #[unstable(feature = "ip_bits", issue = "113744")]
     pub const BITS: u32 = 128;
 
-    /// Converts an IPv6 address into host byte order `u128`.
+    /// Converts an IPv6 address into a `u128` representation using native byte order.
+    ///
+    /// Although IPv6 addresses are big-endian, the `u128` value will use the target platform's
+    /// native byte order. That is, the `u128` value is an integer representation of the IPv6
+    /// address and not an integer interpretation of the IPv6 address's big-endian bitstring. This
+    /// means that the `u128` value masked with `0xffffffffffffffffffffffffffff0000_u128` will set
+    /// the last segment in the address to 0, regardless of the target platform's endianness.
     ///
     /// # Examples
     ///
@@ -1238,6 +1262,24 @@ impl Ipv6Addr {
     /// );
     /// assert_eq!(0x102030405060708090A0B0C0D0E0F00D_u128, u128::from(addr));
     /// ```
+    ///
+    /// ```
+    /// #![feature(ip_bits)]
+    /// use std::net::Ipv6Addr;
+    ///
+    /// let addr = Ipv6Addr::new(
+    ///     0x1020, 0x3040, 0x5060, 0x7080,
+    ///     0x90A0, 0xB0C0, 0xD0E0, 0xF00D,
+    /// );
+    /// let addr_bits = addr.to_bits() & 0xffffffffffffffffffffffffffff0000_u128;
+    /// assert_eq!(
+    ///     Ipv6Addr::new(
+    ///         0x1020, 0x3040, 0x5060, 0x7080,
+    ///         0x90A0, 0xB0C0, 0xD0E0, 0x0000,
+    ///     ),
+    ///     Ipv6Addr::from_bits(addr_bits));
+    ///
+    /// ```
     #[rustc_const_unstable(feature = "ip_bits", issue = "113744")]
     #[unstable(feature = "ip_bits", issue = "113744")]
     #[must_use]
@@ -1246,7 +1288,9 @@ impl Ipv6Addr {
         u128::from_be_bytes(self.octets)
     }
 
-    /// Converts a host byte order `u128` into an IPv6 address.
+    /// Converts a native byte order `u128` into an IPv6 address.
+    ///
+    /// See [`Ipv6Addr::to_bits`] for an explanation on endianness.
     ///
     /// # Examples
     ///
-- 
cgit 1.4.1-3-g733a5


From efe8ae730f4030e544ac733bc20c0825c713e7d9 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Wed, 6 Dec 2023 22:01:21 +0000
Subject: Fix const drop checking

---
 .../src/transform/check_consts/check.rs            | 23 ++++++++++------------
 .../check_consts/post_drop_elaboration.rs          |  5 ++---
 .../src/transform/check_consts/qualifs.rs          | 22 +++++++++++++++++----
 compiler/rustc_middle/src/ty/util.rs               |  6 +++---
 .../src/traits/select/candidate_assembly.rs        | 15 +++++++++++---
 .../src/traits/select/confirmation.rs              | 16 ++++++++-------
 tests/ui/consts/precise-drop-with-promoted.rs      |  7 +------
 tests/ui/consts/precise-drop-with-promoted.stderr  |  6 ------
 .../rfc-2632-const-trait-impl/effects/minicore.rs  |  4 ++--
 .../effects/minicore.stderr                        | 20 -------------------
 10 files changed, 57 insertions(+), 67 deletions(-)
 delete mode 100644 tests/ui/consts/precise-drop-with-promoted.stderr
 delete mode 100644 tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.stderr

diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index bcc42a376ea..e604f4c1aec 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -22,7 +22,7 @@ use std::mem;
 use std::ops::{ControlFlow, Deref};
 
 use super::ops::{self, NonConstOp, Status};
-use super::qualifs::{self, CustomEq, HasMutInterior, NeedsDrop};
+use super::qualifs::{self, CustomEq, HasMutInterior, NeedsDrop, NeedsNonConstDrop};
 use super::resolver::FlowSensitiveAnalysis;
 use super::{ConstCx, Qualif};
 use crate::const_eval::is_unstable_const_fn;
@@ -35,7 +35,7 @@ type QualifResults<'mir, 'tcx, Q> =
 pub struct Qualifs<'mir, 'tcx> {
     has_mut_interior: Option<QualifResults<'mir, 'tcx, HasMutInterior>>,
     needs_drop: Option<QualifResults<'mir, 'tcx, NeedsDrop>>,
-    // needs_non_const_drop: Option<QualifResults<'mir, 'tcx, NeedsNonConstDrop>>,
+    needs_non_const_drop: Option<QualifResults<'mir, 'tcx, NeedsNonConstDrop>>,
 }
 
 impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
@@ -78,27 +78,25 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
         local: Local,
         location: Location,
     ) -> bool {
-        // FIXME(effects) replace with `NeedsNonconstDrop` after const traits work again
-        /*
         let ty = ccx.body.local_decls[local].ty;
-        if !NeedsDrop::in_any_value_of_ty(ccx, ty) {
+        // Peeking into opaque types causes cycles if the current function declares said opaque
+        // type. Thus we avoid short circuiting on the type and instead run the more expensive
+        // analysis that looks at the actual usage within this function
+        if !ty.has_opaque_types() && !NeedsNonConstDrop::in_any_value_of_ty(ccx, ty) {
             return false;
         }
 
         let needs_non_const_drop = self.needs_non_const_drop.get_or_insert_with(|| {
             let ConstCx { tcx, body, .. } = *ccx;
 
-            FlowSensitiveAnalysis::new(NeedsDrop, ccx)
-                .into_engine(tcx, &body)
+            FlowSensitiveAnalysis::new(NeedsNonConstDrop, ccx)
+                .into_engine(tcx, body)
                 .iterate_to_fixpoint()
-                .into_results_cursor(&body)
+                .into_results_cursor(body)
         });
 
         needs_non_const_drop.seek_before_primary_effect(location);
         needs_non_const_drop.get().contains(local)
-        */
-
-        self.needs_drop(ccx, local, location)
     }
 
     /// Returns `true` if `local` is `HasMutInterior` at the given `Location`.
@@ -1013,9 +1011,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 let mut err_span = self.span;
                 let ty_of_dropped_place = dropped_place.ty(self.body, self.tcx).ty;
 
-                // FIXME(effects) replace with `NeedsNonConstDrop` once we fix const traits
                 let ty_needs_non_const_drop =
-                    qualifs::NeedsDrop::in_any_value_of_ty(self.ccx, ty_of_dropped_place);
+                    qualifs::NeedsNonConstDrop::in_any_value_of_ty(self.ccx, ty_of_dropped_place);
 
                 debug!(?ty_of_dropped_place, ?ty_needs_non_const_drop);
 
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
index 06371438ec1..5cd13783c23 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
@@ -5,7 +5,7 @@ use rustc_span::{symbol::sym, Span};
 
 use super::check::Qualifs;
 use super::ops::{self, NonConstOp};
-use super::qualifs::{NeedsDrop, Qualif};
+use super::qualifs::{NeedsNonConstDrop, Qualif};
 use super::ConstCx;
 
 /// Returns `true` if we should use the more precise live drop checker that runs after drop
@@ -83,8 +83,7 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
             mir::TerminatorKind::Drop { place: dropped_place, .. } => {
                 let dropped_ty = dropped_place.ty(self.body, self.tcx).ty;
 
-                // FIXME(effects) use `NeedsNonConstDrop`
-                if !NeedsDrop::in_any_value_of_ty(self.ccx, dropped_ty) {
+                if !NeedsNonConstDrop::in_any_value_of_ty(self.ccx, dropped_ty) {
                     // Instead of throwing a bug, we just return here. This is because we have to
                     // run custom `const Drop` impls.
                     return;
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
index de3186a53c1..d5f418e1710 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
@@ -23,8 +23,7 @@ pub fn in_any_value_of_ty<'tcx>(
     ConstQualifs {
         has_mut_interior: HasMutInterior::in_any_value_of_ty(cx, ty),
         needs_drop: NeedsDrop::in_any_value_of_ty(cx, ty),
-        // FIXME(effects)
-        needs_non_const_drop: NeedsDrop::in_any_value_of_ty(cx, ty),
+        needs_non_const_drop: NeedsNonConstDrop::in_any_value_of_ty(cx, ty),
         custom_eq: CustomEq::in_any_value_of_ty(cx, ty),
         tainted_by_errors,
     }
@@ -155,12 +154,27 @@ impl Qualif for NeedsNonConstDrop {
             return false;
         }
 
-        // FIXME(effects) constness
+        // FIXME(effects): If `destruct` is not a `const_trait`,
+        // or effects are disabled in this crate, then give up.
+        let destruct_def_id = cx.tcx.require_lang_item(LangItem::Destruct, Some(cx.body.span));
+        if cx.tcx.generics_of(destruct_def_id).host_effect_index.is_none()
+            || !cx.tcx.features().effects
+        {
+            return NeedsDrop::in_any_value_of_ty(cx, ty);
+        }
+
         let obligation = Obligation::new(
             cx.tcx,
             ObligationCause::dummy_with_span(cx.body.span),
             cx.param_env,
-            ty::TraitRef::from_lang_item(cx.tcx, LangItem::Destruct, cx.body.span, [ty]),
+            ty::TraitRef::new(
+                cx.tcx,
+                destruct_def_id,
+                [
+                    ty::GenericArg::from(ty),
+                    ty::GenericArg::from(cx.tcx.expected_const_effect_param_for_body(cx.def_id())),
+                ],
+            ),
         );
 
         let infcx = cx.tcx.infer_ctxt().build();
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 70252a4dc67..52c3529d2b4 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -781,9 +781,9 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn expected_const_effect_param_for_body(self, def_id: LocalDefId) -> ty::Const<'tcx> {
-        // if the callee does have the param, we need to equate the param to some const
-        // value no matter whether the effects feature is enabled in the local crate,
-        // because inference will fail if we don't.
+        // FIXME(effects): This is suspicious and should probably not be done,
+        // especially now that we enforce host effects and then properly handle
+        // effect vars during fallback.
         let mut host_always_on =
             !self.features().effects || self.sess.opts.unstable_opts.unleash_the_miri_inside_of_you;
 
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index 5a559bb5fa0..367de517af2 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -872,9 +872,18 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) {
         // If the predicate is `~const Destruct` in a non-const environment, we don't actually need
         // to check anything. We'll short-circuit checking any obligations in confirmation, too.
-        // FIXME(effects)
-        if true {
-            candidates.vec.push(ConstDestructCandidate(None));
+        let Some(host_effect_index) =
+            self.tcx().generics_of(obligation.predicate.def_id()).host_effect_index
+        else {
+            candidates.vec.push(BuiltinCandidate { has_nested: false });
+            return;
+        };
+        // If the obligation has `host = true`, then the obligation is non-const and it's always
+        // trivially implemented.
+        if obligation.predicate.skip_binder().trait_ref.args.const_at(host_effect_index)
+            == self.tcx().consts.true_
+        {
+            candidates.vec.push(BuiltinCandidate { has_nested: false });
             return;
         }
 
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index fc4f6f37621..8567f4f0e70 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -1172,11 +1172,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligation: &PolyTraitObligation<'tcx>,
         impl_def_id: Option<DefId>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
-        // `~const Destruct` in a non-const environment is always trivially true, since our type is `Drop`
-        // FIXME(effects)
-        if true {
-            return Ok(vec![]);
-        }
+        let Some(host_effect_index) =
+            self.tcx().generics_of(obligation.predicate.def_id()).host_effect_index
+        else {
+            bug!()
+        };
+        let host_effect_param: ty::GenericArg<'tcx> =
+            obligation.predicate.skip_binder().trait_ref.args.const_at(host_effect_index).into();
 
         let drop_trait = self.tcx().require_lang_item(LangItem::Drop, None);
 
@@ -1284,7 +1286,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                                 self.tcx(),
                                 LangItem::Destruct,
                                 cause.span,
-                                [nested_ty],
+                                [nested_ty.into(), host_effect_param],
                             ),
                             polarity: ty::ImplPolarity::Positive,
                         }),
@@ -1310,7 +1312,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             self.tcx(),
                             LangItem::Destruct,
                             cause.span,
-                            [nested_ty],
+                            [nested_ty.into(), host_effect_param],
                         ),
                         polarity: ty::ImplPolarity::Positive,
                     });
diff --git a/tests/ui/consts/precise-drop-with-promoted.rs b/tests/ui/consts/precise-drop-with-promoted.rs
index 0c0514dd9d5..7cbe3c4e415 100644
--- a/tests/ui/consts/precise-drop-with-promoted.rs
+++ b/tests/ui/consts/precise-drop-with-promoted.rs
@@ -1,11 +1,6 @@
 // Regression test for issue #89938.
+// check-pass
 // compile-flags: --crate-type=lib
-// known-bug: #103507
-// failure-status: 101
-// normalize-stderr-test "note: .*\n\n" -> ""
-// normalize-stderr-test "thread 'rustc' panicked.*\n.*\n" -> ""
-// normalize-stderr-test "(error: internal compiler error: [^:]+):\d+:\d+: " -> "$1:LL:CC: "
-// rustc-env:RUST_BACKTRACE=0
 
 #![feature(const_precise_live_drops)]
 
diff --git a/tests/ui/consts/precise-drop-with-promoted.stderr b/tests/ui/consts/precise-drop-with-promoted.stderr
deleted file mode 100644
index a56672048eb..00000000000
--- a/tests/ui/consts/precise-drop-with-promoted.stderr
+++ /dev/null
@@ -1,6 +0,0 @@
-error: the compiler unexpectedly panicked. this is a bug.
-
-query stack during panic:
-#0 [mir_drops_elaborated_and_const_checked] elaborating drops for `f`
-#1 [analysis] running analysis passes on this crate
-end of query stack
diff --git a/tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.rs b/tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.rs
index 2c6fd83484f..2a2e8cec3f0 100644
--- a/tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.rs
+++ b/tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.rs
@@ -1,3 +1,5 @@
+// check-pass
+
 #![crate_type = "lib"]
 #![feature(no_core, lang_items, unboxed_closures, auto_traits, intrinsics, rustc_attrs)]
 #![feature(fundamental)]
@@ -6,8 +8,6 @@
 #![no_std]
 #![no_core]
 
-// known-bug: #110395
-
 #[lang = "sized"]
 trait Sized {}
 #[lang = "copy"]
diff --git a/tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.stderr b/tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.stderr
deleted file mode 100644
index 3c1e6dda85c..00000000000
--- a/tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.stderr
+++ /dev/null
@@ -1,20 +0,0 @@
-error[E0493]: destructor of `Self` cannot be evaluated at compile-time
-  --> $DIR/minicore.rs:501:9
-   |
-LL |         *self = source.clone()
-   |         ^^^^^
-   |         |
-   |         the destructor for this type cannot be evaluated in constant functions
-   |         value is dropped here
-
-error[E0493]: destructor of `T` cannot be evaluated at compile-time
-  --> $DIR/minicore.rs:511:35
-   |
-LL | const fn drop<T: ~const Destruct>(_: T) {}
-   |                                   ^      - value is dropped here
-   |                                   |
-   |                                   the destructor for this type cannot be evaluated in constant functions
-
-error: aborting due to 2 previous errors
-
-For more information about this error, try `rustc --explain E0493`.
-- 
cgit 1.4.1-3-g733a5


From 3978f545ba25888066abcd1770514294cff42c9f Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Thu, 7 Dec 2023 17:52:51 +0100
Subject: add unused `NormalizesTo` predicate

---
 compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs     | 12 +---
 compiler/rustc_infer/src/traits/util.rs            | 70 ++++++++--------------
 compiler/rustc_middle/src/ty/context.rs            |  1 +
 compiler/rustc_middle/src/ty/flags.rs              |  4 ++
 compiler/rustc_middle/src/ty/mod.rs                | 39 ++++++++++++
 compiler/rustc_middle/src/ty/print/pretty.rs       |  1 +
 compiler/rustc_middle/src/ty/structural_impls.rs   |  6 ++
 compiler/rustc_smir/src/rustc_smir/convert/ty.rs   |  1 +
 .../src/solve/eval_ctxt/mod.rs                     |  1 +
 .../rustc_trait_selection/src/solve/fulfill.rs     |  5 ++
 .../rustc_trait_selection/src/traits/auto_trait.rs |  1 +
 .../traits/error_reporting/type_err_ctxt_ext.rs    |  5 ++
 .../rustc_trait_selection/src/traits/fulfill.rs    | 10 +++-
 .../query/type_op/implied_outlives_bounds.rs       | 13 ++--
 .../rustc_trait_selection/src/traits/select/mod.rs |  5 +-
 .../rustc_traits/src/normalize_erasing_regions.rs  |  1 +
 compiler/rustc_type_ir/src/interner.rs             |  1 +
 compiler/rustc_type_ir/src/predicate_kind.rs       | 16 +++++
 18 files changed, 128 insertions(+), 64 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
index e1a2a260df7..1f2bd92a15c 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
@@ -657,19 +657,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 | ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(..))
                 | ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..))
                 | ty::PredicateKind::ObjectSafe(..)
+                | ty::PredicateKind::NormalizesTo(..)
                 | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..))
                 | ty::PredicateKind::ConstEquate(..)
-                // N.B., this predicate is created by breaking down a
-                // `ClosureType: FnFoo()` predicate, where
-                // `ClosureType` represents some `Closure`. It can't
-                // possibly be referring to the current closure,
-                // because we haven't produced the `Closure` for
-                // this closure yet; this is exactly why the other
-                // code is looking for a self type of an unresolved
-                // inference variable.
-                | ty::PredicateKind::Ambiguous
-                 => None,
+                | ty::PredicateKind::Ambiguous => None,
             },
         )
     }
diff --git a/compiler/rustc_infer/src/traits/util.rs b/compiler/rustc_infer/src/traits/util.rs
index 1bcae736fd9..50190058a76 100644
--- a/compiler/rustc_infer/src/traits/util.rs
+++ b/compiler/rustc_infer/src/traits/util.rs
@@ -261,9 +261,14 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
     fn elaborate(&mut self, elaboratable: &O) {
         let tcx = self.visited.tcx;
 
-        let bound_predicate = elaboratable.predicate().kind();
-        match bound_predicate.skip_binder() {
-            ty::PredicateKind::Clause(ty::ClauseKind::Trait(data)) => {
+        // We only elaborate clauses.
+        let Some(clause) = elaboratable.predicate().as_clause() else {
+            return;
+        };
+
+        let bound_clause = clause.kind();
+        match bound_clause.skip_binder() {
+            ty::ClauseKind::Trait(data) => {
                 // Negative trait bounds do not imply any supertrait bounds
                 if data.polarity == ty::ImplPolarity::Negative {
                     return;
@@ -280,49 +285,16 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
                 let obligations =
                     predicates.predicates.iter().enumerate().map(|(index, &(clause, span))| {
                         elaboratable.child_with_derived_cause(
-                            clause.subst_supertrait(tcx, &bound_predicate.rebind(data.trait_ref)),
+                            clause.subst_supertrait(tcx, &bound_clause.rebind(data.trait_ref)),
                             span,
-                            bound_predicate.rebind(data),
+                            bound_clause.rebind(data),
                             index,
                         )
                     });
                 debug!(?data, ?obligations, "super_predicates");
                 self.extend_deduped(obligations);
             }
-            ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..)) => {
-                // Currently, we do not elaborate WF predicates,
-                // although we easily could.
-            }
-            ty::PredicateKind::ObjectSafe(..) => {
-                // Currently, we do not elaborate object-safe
-                // predicates.
-            }
-            ty::PredicateKind::Subtype(..) => {
-                // Currently, we do not "elaborate" predicates like `X <: Y`,
-                // though conceivably we might.
-            }
-            ty::PredicateKind::Coerce(..) => {
-                // Currently, we do not "elaborate" predicates like `X -> Y`,
-                // though conceivably we might.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::Projection(..)) => {
-                // Nothing to elaborate in a projection predicate.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..)) => {
-                // Currently, we do not elaborate const-evaluatable
-                // predicates.
-            }
-            ty::PredicateKind::ConstEquate(..) => {
-                // Currently, we do not elaborate const-equate
-                // predicates.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(..)) => {
-                // Nothing to elaborate from `'a: 'b`.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(
-                ty_max,
-                r_min,
-            ))) => {
+            ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(ty_max, r_min)) => {
                 // We know that `T: 'a` for some type `T`. We can
                 // often elaborate this. For example, if we know that
                 // `[U]: 'a`, that implies that `U: 'a`. Similarly, if
@@ -385,15 +357,25 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
                             }
                         })
                         .map(|clause| {
-                            elaboratable.child(bound_predicate.rebind(clause).to_predicate(tcx))
+                            elaboratable.child(bound_clause.rebind(clause).to_predicate(tcx))
                         }),
                 );
             }
-            ty::PredicateKind::Ambiguous => {}
-            ty::PredicateKind::AliasRelate(..) => {
-                // No
+            ty::ClauseKind::RegionOutlives(..) => {
+                // Nothing to elaborate from `'a: 'b`.
+            }
+            ty::ClauseKind::WellFormed(..) => {
+                // Currently, we do not elaborate WF predicates,
+                // although we easily could.
+            }
+            ty::ClauseKind::Projection(..) => {
+                // Nothing to elaborate in a projection predicate.
+            }
+            ty::ClauseKind::ConstEvaluatable(..) => {
+                // Currently, we do not elaborate const-evaluatable
+                // predicates.
             }
-            ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..)) => {
+            ty::ClauseKind::ConstArgHasType(..) => {
                 // Nothing to elaborate
             }
         }
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 4a01a24fd61..3012434ad3f 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -121,6 +121,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type RegionOutlivesPredicate = ty::RegionOutlivesPredicate<'tcx>;
     type TypeOutlivesPredicate = ty::TypeOutlivesPredicate<'tcx>;
     type ProjectionPredicate = ty::ProjectionPredicate<'tcx>;
+    type NormalizesTo = ty::NormalizesTo<'tcx>;
     type SubtypePredicate = ty::SubtypePredicate<'tcx>;
     type CoercePredicate = ty::CoercePredicate<'tcx>;
     type ClosureKind = ty::ClosureKind;
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index 5084fc98913..f9a2385b100 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -272,6 +272,10 @@ impl FlagComputation {
                 self.add_const(found);
             }
             ty::PredicateKind::Ambiguous => {}
+            ty::PredicateKind::NormalizesTo(ty::NormalizesTo { alias, term }) => {
+                self.add_alias_ty(alias);
+                self.add_term(term);
+            }
             ty::PredicateKind::AliasRelate(t1, t2, _) => {
                 self.add_term(t1);
                 self.add_term(t2);
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 77196486ac1..203c9eb65df 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -553,6 +553,10 @@ impl<'tcx> Predicate<'tcx> {
     pub fn allow_normalization(self) -> bool {
         match self.kind().skip_binder() {
             PredicateKind::Clause(ClauseKind::WellFormed(_)) => false,
+            // `NormalizesTo` is only used in the new solver, so this shouldn't
+            // matter. Normalizing `term` would be 'wrong' however, as it changes whether
+            // `normalizes-to(<T as Trait>::Assoc, <T as Trait>::Assoc)` holds.
+            PredicateKind::NormalizesTo(..) => false,
             PredicateKind::Clause(ClauseKind::Trait(_))
             | PredicateKind::Clause(ClauseKind::RegionOutlives(_))
             | PredicateKind::Clause(ClauseKind::TypeOutlives(_))
@@ -1093,6 +1097,33 @@ impl<'tcx> PolyProjectionPredicate<'tcx> {
     }
 }
 
+/// Used by the new solver. Unlike a `ProjectionPredicate` this can only be
+/// proven by actually normalizing `alias`.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
+pub struct NormalizesTo<'tcx> {
+    pub alias: AliasTy<'tcx>,
+    pub term: Term<'tcx>,
+}
+
+impl<'tcx> NormalizesTo<'tcx> {
+    pub fn self_ty(self) -> Ty<'tcx> {
+        self.alias.self_ty()
+    }
+
+    pub fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> NormalizesTo<'tcx> {
+        Self { alias: self.alias.with_self_ty(tcx, self_ty), ..self }
+    }
+
+    pub fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId {
+        self.alias.trait_def_id(tcx)
+    }
+
+    pub fn def_id(self) -> DefId {
+        self.alias.def_id
+    }
+}
+
 pub trait ToPolyTraitRef<'tcx> {
     fn to_poly_trait_ref(&self) -> PolyTraitRef<'tcx>;
 }
@@ -1274,6 +1305,12 @@ impl<'tcx> ToPredicate<'tcx, Clause<'tcx>> for PolyProjectionPredicate<'tcx> {
     }
 }
 
+impl<'tcx> ToPredicate<'tcx> for NormalizesTo<'tcx> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        PredicateKind::NormalizesTo(self).to_predicate(tcx)
+    }
+}
+
 impl<'tcx> Predicate<'tcx> {
     pub fn to_opt_poly_trait_pred(self) -> Option<PolyTraitPredicate<'tcx>> {
         let predicate = self.kind();
@@ -1281,6 +1318,7 @@ impl<'tcx> Predicate<'tcx> {
             PredicateKind::Clause(ClauseKind::Trait(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(ClauseKind::Projection(..))
             | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
+            | PredicateKind::NormalizesTo(..)
             | PredicateKind::AliasRelate(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
@@ -1300,6 +1338,7 @@ impl<'tcx> Predicate<'tcx> {
             PredicateKind::Clause(ClauseKind::Projection(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(ClauseKind::Trait(..))
             | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
+            | PredicateKind::NormalizesTo(..)
             | PredicateKind::AliasRelate(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 63b706e6b3d..faf2a854f69 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -2814,6 +2814,7 @@ define_print! {
                 p!("the constant `", print(c1), "` equals `", print(c2), "`")
             }
             ty::PredicateKind::Ambiguous => p!("ambiguous"),
+            ty::PredicateKind::NormalizesTo(data) => p!(print(data.alias), " normalizes-to ", print(data.term)),
             ty::PredicateKind::AliasRelate(t1, t2, dir) => p!(print(t1), write(" {} ", dir), print(t2)),
         }
     }
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 971acda33e2..51dc8f7c5a9 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -173,6 +173,12 @@ impl<'tcx> fmt::Debug for ty::ProjectionPredicate<'tcx> {
     }
 }
 
+impl<'tcx> fmt::Debug for ty::NormalizesTo<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "NormalizesTo({:?}, {:?})", self.alias, self.term)
+    }
+}
+
 impl<'tcx> fmt::Debug for ty::Predicate<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{:?}", self.kind())
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
index 4fe847c291c..cbdddc30072 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
@@ -601,6 +601,7 @@ impl<'tcx> Stable<'tcx> for ty::PredicateKind<'tcx> {
                 stable_mir::ty::PredicateKind::ConstEquate(a.stable(tables), b.stable(tables))
             }
             PredicateKind::Ambiguous => stable_mir::ty::PredicateKind::Ambiguous,
+            PredicateKind::NormalizesTo(_pred) => unimplemented!(),
             PredicateKind::AliasRelate(a, b, alias_relation_direction) => {
                 stable_mir::ty::PredicateKind::AliasRelate(
                     a.unpack().stable(tables),
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
index 5d404d7e3e9..348dfdf725f 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
@@ -423,6 +423,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
                 ty::PredicateKind::ConstEquate(_, _) => {
                     bug!("ConstEquate should not be emitted when `-Ztrait-solver=next` is active")
                 }
+                ty::PredicateKind::NormalizesTo(_) => unimplemented!(),
                 ty::PredicateKind::AliasRelate(lhs, rhs, direction) => self
                     .compute_alias_relate_goal(Goal {
                         param_env,
diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs
index b73ec93b824..2139210b873 100644
--- a/compiler/rustc_trait_selection/src/solve/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs
@@ -108,6 +108,11 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
                                             MismatchedProjectionTypes { err: TypeError::Mismatch },
                                         )
                                     }
+                                    ty::PredicateKind::NormalizesTo(..) => {
+                                        FulfillmentErrorCode::CodeProjectionError(
+                                            MismatchedProjectionTypes { err: TypeError::Mismatch },
+                                        )
+                                    }
                                     ty::PredicateKind::AliasRelate(_, _, _) => {
                                         FulfillmentErrorCode::CodeProjectionError(
                                             MismatchedProjectionTypes { err: TypeError::Mismatch },
diff --git a/compiler/rustc_trait_selection/src/traits/auto_trait.rs b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
index d8bf97138cd..0b8e6e2ef8b 100644
--- a/compiler/rustc_trait_selection/src/traits/auto_trait.rs
+++ b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
@@ -820,6 +820,7 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                 // the `ParamEnv`.
                 ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..))
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..))
+                | ty::PredicateKind::NormalizesTo(..)
                 | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::ObjectSafe(..)
                 | ty::PredicateKind::Subtype(..)
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index 090706070eb..1f94fbaf9f8 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -854,6 +854,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
                     ty::PredicateKind::Ambiguous => span_bug!(span, "ambiguous"),
 
+                    ty::PredicateKind::NormalizesTo(..) => span_bug!(
+                        span,
+                        "NormalizesTo predicate should never be the predicate cause of a SelectionError"
+                    ),
+
                     ty::PredicateKind::AliasRelate(..) => span_bug!(
                         span,
                         "AliasRelate predicate should never be the predicate cause of a SelectionError"
diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs
index fd39fce9dd1..9cbddd2bb2b 100644
--- a/compiler/rustc_trait_selection/src/traits/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs
@@ -360,8 +360,11 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                     ProcessResult::Changed(mk_pending(vec![obligation.with(infcx.tcx, pred)]))
                 }
                 ty::PredicateKind::Ambiguous => ProcessResult::Unchanged,
+                ty::PredicateKind::NormalizesTo(..) => {
+                    bug!("NormalizesTo is only used by the new solver")
+                }
                 ty::PredicateKind::AliasRelate(..) => {
-                    bug!("AliasRelate is only used for new solver")
+                    bug!("AliasRelate is only used by the new solver")
                 }
             },
             Some(pred) => match pred {
@@ -412,8 +415,11 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 }
 
                 ty::PredicateKind::Ambiguous => ProcessResult::Unchanged,
+                ty::PredicateKind::NormalizesTo(..) => {
+                    bug!("NormalizesTo is only used by the new solver")
+                }
                 ty::PredicateKind::AliasRelate(..) => {
-                    bug!("AliasRelate is only used for new solver")
+                    bug!("AliasRelate is only used by the new solver")
                 }
 
                 // General case overflow check. Allow `process_trait_obligation`
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
index 56844f39478..f9b8ea32d89 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
@@ -123,9 +123,9 @@ pub fn compute_implied_outlives_bounds_inner<'tcx>(
                 Some(pred) => pred,
             };
             match pred {
-                ty::PredicateKind::Clause(ty::ClauseKind::Trait(..))
                 // FIXME(const_generics): Make sure that `<'a, 'b, const N: &'a &'b u32>` is sound
                 // if we ever support that
+                ty::PredicateKind::Clause(ty::ClauseKind::Trait(..))
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..))
                 | ty::PredicateKind::Subtype(..)
                 | ty::PredicateKind::Coerce(..)
@@ -134,8 +134,8 @@ pub fn compute_implied_outlives_bounds_inner<'tcx>(
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..))
                 | ty::PredicateKind::ConstEquate(..)
                 | ty::PredicateKind::Ambiguous
-                | ty::PredicateKind::AliasRelate(..)
-                 => {}
+                | ty::PredicateKind::NormalizesTo(..)
+                | ty::PredicateKind::AliasRelate(..) => {}
 
                 // We need to search through *all* WellFormed predicates
                 ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(arg)) => {
@@ -143,10 +143,9 @@ pub fn compute_implied_outlives_bounds_inner<'tcx>(
                 }
 
                 // We need to register region relationships
-                ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(
-                    r_a,
-                    r_b,
-                ))) => outlives_bounds.push(ty::OutlivesPredicate(r_a.into(), r_b)),
+                ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(
+                    ty::OutlivesPredicate(r_a, r_b),
+                )) => outlives_bounds.push(ty::OutlivesPredicate(r_a.into(), r_b)),
 
                 ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(
                     ty_a,
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 0d3ec41f511..6ba379467da 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -990,8 +990,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                         }
                     }
                 }
+                ty::PredicateKind::NormalizesTo(..) => {
+                    bug!("NormalizesTo is only used by the new solver")
+                }
                 ty::PredicateKind::AliasRelate(..) => {
-                    bug!("AliasRelate is only used for new solver")
+                    bug!("AliasRelate is only used by the new solver")
                 }
                 ty::PredicateKind::Ambiguous => Ok(EvaluatedToAmbig),
                 ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(ct, ty)) => {
diff --git a/compiler/rustc_traits/src/normalize_erasing_regions.rs b/compiler/rustc_traits/src/normalize_erasing_regions.rs
index 06486a100a9..0576fe01027 100644
--- a/compiler/rustc_traits/src/normalize_erasing_regions.rs
+++ b/compiler/rustc_traits/src/normalize_erasing_regions.rs
@@ -55,6 +55,7 @@ fn not_outlives_predicate(p: ty::Predicate<'_>) -> bool {
         ty::PredicateKind::Clause(ty::ClauseKind::Trait(..))
         | ty::PredicateKind::Clause(ty::ClauseKind::Projection(..))
         | ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..))
+        | ty::PredicateKind::NormalizesTo(..)
         | ty::PredicateKind::AliasRelate(..)
         | ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..))
         | ty::PredicateKind::ObjectSafe(..)
diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs
index 16508c1a257..300b0bf090d 100644
--- a/compiler/rustc_type_ir/src/interner.rs
+++ b/compiler/rustc_type_ir/src/interner.rs
@@ -57,6 +57,7 @@ pub trait Interner: Sized {
     type RegionOutlivesPredicate: Clone + Debug + Hash + Eq;
     type TypeOutlivesPredicate: Clone + Debug + Hash + Eq;
     type ProjectionPredicate: Clone + Debug + Hash + Eq;
+    type NormalizesTo: Clone + Debug + Hash + Eq;
     type SubtypePredicate: Clone + Debug + Hash + Eq;
     type CoercePredicate: Clone + Debug + Hash + Eq;
     type ClosureKind: Clone + Debug + Hash + Eq;
diff --git a/compiler/rustc_type_ir/src/predicate_kind.rs b/compiler/rustc_type_ir/src/predicate_kind.rs
index f6a2dca4eee..adeb3f57497 100644
--- a/compiler/rustc_type_ir/src/predicate_kind.rs
+++ b/compiler/rustc_type_ir/src/predicate_kind.rs
@@ -153,6 +153,15 @@ pub enum PredicateKind<I: Interner> {
     /// Used for coherence to mark opaque types as possibly equal to each other but ambiguous.
     Ambiguous,
 
+    /// The alias normalizes to `term`. Unlike `Projection`, this always fails if the alias
+    /// cannot be normalized in the current context.
+    ///
+    /// `Projection(<T as Trait>::Assoc, ?x)` results in `?x == <T as Trait>::Assoc` while
+    /// `NormalizesTo(<T as Trait>::Assoc, ?x)` results in `NoSolution`.
+    ///
+    /// Only used in the new solver.
+    NormalizesTo(I::NormalizesTo),
+
     /// Separate from `ClauseKind::Projection` which is used for normalization in new solver.
     /// This predicate requires two terms to be equal to eachother.
     ///
@@ -160,6 +169,7 @@ pub enum PredicateKind<I: Interner> {
     AliasRelate(I::Term, I::Term, AliasRelationDirection),
 }
 
+/// FIXME: This impl sh
 impl<I: Interner> Copy for PredicateKind<I>
 where
     I::DefId: Copy,
@@ -168,6 +178,7 @@ where
     I::Term: Copy,
     I::CoercePredicate: Copy,
     I::SubtypePredicate: Copy,
+    I::NormalizesTo: Copy,
     ClauseKind<I>: Copy,
 {
 }
@@ -198,6 +209,7 @@ where
     I::Term: TypeFoldable<I>,
     I::CoercePredicate: TypeFoldable<I>,
     I::SubtypePredicate: TypeFoldable<I>,
+    I::NormalizesTo: TypeFoldable<I>,
     ClauseKind<I>: TypeFoldable<I>,
 {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
@@ -210,6 +222,7 @@ where
                 PredicateKind::ConstEquate(a.try_fold_with(folder)?, b.try_fold_with(folder)?)
             }
             PredicateKind::Ambiguous => PredicateKind::Ambiguous,
+            PredicateKind::NormalizesTo(p) => PredicateKind::NormalizesTo(p.try_fold_with(folder)?),
             PredicateKind::AliasRelate(a, b, d) => PredicateKind::AliasRelate(
                 a.try_fold_with(folder)?,
                 b.try_fold_with(folder)?,
@@ -227,6 +240,7 @@ where
     I::Term: TypeVisitable<I>,
     I::CoercePredicate: TypeVisitable<I>,
     I::SubtypePredicate: TypeVisitable<I>,
+    I::NormalizesTo: TypeVisitable<I>,
     ClauseKind<I>: TypeVisitable<I>,
 {
     fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
@@ -240,6 +254,7 @@ where
                 b.visit_with(visitor)
             }
             PredicateKind::Ambiguous => ControlFlow::Continue(()),
+            PredicateKind::NormalizesTo(p) => p.visit_with(visitor),
             PredicateKind::AliasRelate(a, b, d) => {
                 a.visit_with(visitor)?;
                 b.visit_with(visitor)?;
@@ -294,6 +309,7 @@ impl<I: Interner> fmt::Debug for PredicateKind<I> {
             }
             PredicateKind::ConstEquate(c1, c2) => write!(f, "ConstEquate({c1:?}, {c2:?})"),
             PredicateKind::Ambiguous => write!(f, "Ambiguous"),
+            PredicateKind::NormalizesTo(p) => p.fmt(f),
             PredicateKind::AliasRelate(t1, t2, dir) => {
                 write!(f, "AliasRelate({t1:?}, {dir:?}, {t2:?})")
             }
-- 
cgit 1.4.1-3-g733a5


From 365aaa8011c6a0603506d20dcc30117555f6a981 Mon Sep 17 00:00:00 2001
From: John Kåre Alsaker <john.kare.alsaker@gmail.com>
Date: Tue, 7 Nov 2023 20:06:39 +0100
Subject: Explicitly implement `DynSync` and `DynSend` for `TyCtxt`

---
 compiler/rustc_middle/src/ty/context.rs | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 4a01a24fd61..36eb70df7d1 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -39,7 +39,9 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
-use rustc_data_structures::sync::{FreezeReadGuard, Lock, WorkerLocal};
+use rustc_data_structures::sync::{self, FreezeReadGuard, Lock, WorkerLocal};
+#[cfg(parallel_compiler)]
+use rustc_data_structures::sync::{DynSend, DynSync};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{
     DecorateLint, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, MultiSpan,
@@ -552,6 +554,16 @@ pub struct TyCtxt<'tcx> {
     gcx: &'tcx GlobalCtxt<'tcx>,
 }
 
+// Explicitly implement `DynSync` and `DynSend` for `TyCtxt` to short circuit trait resolution.
+#[cfg(parallel_compiler)]
+unsafe impl DynSend for TyCtxt<'_> {}
+#[cfg(parallel_compiler)]
+unsafe impl DynSync for TyCtxt<'_> {}
+fn _assert_tcx_fields() {
+    sync::assert_dyn_sync::<&'_ GlobalCtxt<'_>>();
+    sync::assert_dyn_send::<&'_ GlobalCtxt<'_>>();
+}
+
 impl<'tcx> Deref for TyCtxt<'tcx> {
     type Target = &'tcx GlobalCtxt<'tcx>;
     #[inline(always)]
-- 
cgit 1.4.1-3-g733a5


From 6a0a89af804fc93c0017486c81b191958e4b1492 Mon Sep 17 00:00:00 2001
From: Michael Howell <michael@notriddle.com>
Date: Thu, 7 Dec 2023 12:49:17 -0700
Subject: rustdoc: remove unused parameter `reversed` from onEach(Lazy)

This feature was added in edec5807ac5ba90cbc0c61a5ec7b80f29e1eea33
to support JavaScript-based toggles that were later replaced with
HTML `<details>`.
---
 src/librustdoc/html/static/js/storage.js | 25 ++++++-------------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/src/librustdoc/html/static/js/storage.js b/src/librustdoc/html/static/js/storage.js
index c69641092ab..37250ba5a1f 100644
--- a/src/librustdoc/html/static/js/storage.js
+++ b/src/librustdoc/html/static/js/storage.js
@@ -51,22 +51,11 @@ function removeClass(elem, className) {
  * Run a callback for every element of an Array.
  * @param {Array<?>}    arr        - The array to iterate over
  * @param {function(?)} func       - The callback
- * @param {boolean}     [reversed] - Whether to iterate in reverse
  */
-function onEach(arr, func, reversed) {
-    if (arr && arr.length > 0) {
-        if (reversed) {
-            for (let i = arr.length - 1; i >= 0; --i) {
-                if (func(arr[i])) {
-                    return true;
-                }
-            }
-        } else {
-            for (const elem of arr) {
-                if (func(elem)) {
-                    return true;
-                }
-            }
+function onEach(arr, func) {
+    for (const elem of arr) {
+        if (func(elem)) {
+            return true;
         }
     }
     return false;
@@ -80,14 +69,12 @@ function onEach(arr, func, reversed) {
  * https://developer.mozilla.org/en-US/docs/Web/API/NodeList
  * @param {NodeList<?>|HTMLCollection<?>} lazyArray  - An array to iterate over
  * @param {function(?)}                   func       - The callback
- * @param {boolean}                       [reversed] - Whether to iterate in reverse
  */
 // eslint-disable-next-line no-unused-vars
-function onEachLazy(lazyArray, func, reversed) {
+function onEachLazy(lazyArray, func) {
     return onEach(
         Array.prototype.slice.call(lazyArray),
-        func,
-        reversed);
+        func);
 }
 
 function updateLocalStorage(name, value) {
-- 
cgit 1.4.1-3-g733a5


From 88fccc465f157ecc84ec8df9af724fd7d155d534 Mon Sep 17 00:00:00 2001
From: Ian Rees <code@ianrees.nz>
Date: Mon, 4 Dec 2023 15:18:58 +1300
Subject: OnceLock: Rework example, statics aren't dropped

---
 library/std/src/sync/once_lock.rs | 39 +++++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/library/std/src/sync/once_lock.rs b/library/std/src/sync/once_lock.rs
index 52a43913243..b8873a3b59a 100644
--- a/library/std/src/sync/once_lock.rs
+++ b/library/std/src/sync/once_lock.rs
@@ -17,25 +17,36 @@ use crate::sync::Once;
 /// ‘lazy static’ or ‘memoizing’):
 ///
 /// ```
-/// use std::collections::HashMap;
 /// use std::sync::OnceLock;
 ///
-/// fn hash_map() -> &'static HashMap<u32, char> {
-///     static HASHMAP: OnceLock<HashMap<u32, char>> = OnceLock::new();
-///     HASHMAP.get_or_init(|| {
-///         let mut m = HashMap::new();
-///         m.insert(0, 'a');
-///         m.insert(1, 'b');
-///         m.insert(2, 'c');
-///         m
-///     })
+/// struct DeepThought {
+///     answer: String,
 /// }
 ///
-/// // The `HashMap` is built, stored in the `OnceLock`, and returned.
-/// let _ = hash_map();
+/// impl DeepThought {
+/// #   fn great_question() -> String {
+/// #       "42".to_string()
+/// #   }
+/// #
+///     fn new() -> Self {
+///         Self {
+///             // M3 Ultra takes about 16 million years in --release config
+///             answer: Self::great_question(),
+///         }
+///     }
+/// }
+///
+/// fn computation() -> &'static DeepThought {
+///     // n.b. static items do not call [`Drop`] on program termination, so if
+///     // [`DeepThought`] impls Drop, that will not be used for this instance.
+///     static COMPUTATION: OnceLock<DeepThought> = OnceLock::new();
+///     COMPUTATION.get_or_init(|| DeepThought::new())
+/// }
 ///
-/// // The `HashMap` is retrieved from the `OnceLock` and returned.
-/// let _ = hash_map();
+/// // The `DeepThought` is built, stored in the `OnceLock`, and returned.
+/// let _ = computation().answer;
+/// // The `DeepThought` is retrieved from the `OnceLock` and returned.
+/// let _ = computation().answer;
 /// ```
 ///
 /// Writing to a `OnceLock` from a separate thread:
-- 
cgit 1.4.1-3-g733a5


From f312775e4fe78278b48df8f016418adf2c5c413d Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Sun, 26 Nov 2023 06:58:05 +1100
Subject: Remove unused arguments from
 `ResultsVisitor::visit_block_{start,end}`.

---
 compiler/rustc_mir_dataflow/src/framework/direction.rs |  8 ++++----
 compiler/rustc_mir_dataflow/src/framework/graphviz.rs  | 16 ++--------------
 compiler/rustc_mir_dataflow/src/framework/visitor.rs   | 18 ++----------------
 3 files changed, 8 insertions(+), 34 deletions(-)

diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs
index 08b7b1a2619..4c3fadf487b 100644
--- a/compiler/rustc_mir_dataflow/src/framework/direction.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs
@@ -196,7 +196,7 @@ impl Direction for Backward {
     {
         results.reset_to_block_entry(state, block);
 
-        vis.visit_block_end(results, state, block_data, block);
+        vis.visit_block_end(state);
 
         // Terminator
         let loc = Location { block, statement_index: block_data.statements.len() };
@@ -214,7 +214,7 @@ impl Direction for Backward {
             vis.visit_statement_after_primary_effect(results, state, stmt, loc);
         }
 
-        vis.visit_block_start(results, state, block_data, block);
+        vis.visit_block_start(state);
     }
 
     fn join_state_into_successors_of<'tcx, A>(
@@ -449,7 +449,7 @@ impl Direction for Forward {
     {
         results.reset_to_block_entry(state, block);
 
-        vis.visit_block_start(results, state, block_data, block);
+        vis.visit_block_start(state);
 
         for (statement_index, stmt) in block_data.statements.iter().enumerate() {
             let loc = Location { block, statement_index };
@@ -466,7 +466,7 @@ impl Direction for Forward {
         results.reconstruct_terminator_effect(state, term, loc);
         vis.visit_terminator_after_primary_effect(results, state, term, loc);
 
-        vis.visit_block_end(results, state, block_data, block);
+        vis.visit_block_end(state);
     }
 
     fn join_state_into_successors_of<'tcx, A>(
diff --git a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
index fa16cac3168..0270e059a58 100644
--- a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
@@ -545,25 +545,13 @@ where
 {
     type FlowState = A::Domain;
 
-    fn visit_block_start(
-        &mut self,
-        _results: &mut Results<'tcx, A>,
-        state: &Self::FlowState,
-        _block_data: &mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
+    fn visit_block_start(&mut self, state: &Self::FlowState) {
         if A::Direction::IS_FORWARD {
             self.prev_state.clone_from(state);
         }
     }
 
-    fn visit_block_end(
-        &mut self,
-        _results: &mut Results<'tcx, A>,
-        state: &Self::FlowState,
-        _block_data: &mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
+    fn visit_block_end(&mut self, state: &Self::FlowState) {
         if A::Direction::IS_BACKWARD {
             self.prev_state.clone_from(state);
         }
diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs
index e3648bb4076..52cf87b676d 100644
--- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs
@@ -31,14 +31,7 @@ pub fn visit_results<'mir, 'tcx, F, R>(
 pub trait ResultsVisitor<'mir, 'tcx, R> {
     type FlowState;
 
-    fn visit_block_start(
-        &mut self,
-        _results: &mut R,
-        _state: &Self::FlowState,
-        _block_data: &'mir mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
-    }
+    fn visit_block_start(&mut self, _state: &Self::FlowState) {}
 
     /// Called with the `before_statement_effect` of the given statement applied to `state` but not
     /// its `statement_effect`.
@@ -86,14 +79,7 @@ pub trait ResultsVisitor<'mir, 'tcx, R> {
     ) {
     }
 
-    fn visit_block_end(
-        &mut self,
-        _results: &mut R,
-        _state: &Self::FlowState,
-        _block_data: &'mir mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
-    }
+    fn visit_block_end(&mut self, _state: &Self::FlowState) {}
 }
 
 /// Things that can be visited by a `ResultsVisitor`.
-- 
cgit 1.4.1-3-g733a5


From 60e7c6898b2c8e67ec8754cd31f686217d4a944f Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Sun, 26 Nov 2023 08:31:19 +1100
Subject: Remove `impl_visitable!`.

It is used just once. With it removed, the relevant code is a little
boilerplate-y but much easier to read, and is the same length. Overall I
think it's an improvement.
---
 compiler/rustc_borrowck/src/dataflow.rs            | 127 ++++++++++-----------
 .../rustc_mir_dataflow/src/framework/visitor.rs    |   4 +-
 2 files changed, 64 insertions(+), 67 deletions(-)

diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index 54f161ea9b4..5a42d9325fa 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -36,76 +36,73 @@ pub type BorrowckResults<'mir, 'tcx> = BorrowckAnalyses<
 pub type BorrowckFlowState<'mir, 'tcx> =
     <BorrowckResults<'mir, 'tcx> as ResultsVisitable<'tcx>>::FlowState;
 
-macro_rules! impl_visitable {
-    ( $(
-        $T:ident { $( $field:ident : $A:ident ),* $(,)? }
-    )* ) => { $(
-        impl<'tcx, $($A),*, D: Direction> ResultsVisitable<'tcx> for $T<$( Results<'tcx, $A> ),*>
-        where
-            $( $A: Analysis<'tcx, Direction = D>, )*
-        {
-            type Direction = D;
-            type FlowState = $T<$( $A::Domain ),*>;
-
-            fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
-                $T {
-                    $( $field: self.$field.analysis.bottom_value(body) ),*
-                }
-            }
-
-            fn reset_to_block_entry(
-                &self,
-                state: &mut Self::FlowState,
-                block: BasicBlock,
-            ) {
-                $( state.$field.clone_from(&self.$field.entry_set_for_block(block)); )*
-            }
+impl<'tcx, B, U, E, D: Direction> ResultsVisitable<'tcx>
+    for BorrowckAnalyses<Results<'tcx, B>, Results<'tcx, U>, Results<'tcx, E>>
+where
+    B: Analysis<'tcx, Direction = D>,
+    U: Analysis<'tcx, Direction = D>,
+    E: Analysis<'tcx, Direction = D>,
+{
+    type Direction = D;
+    type FlowState = BorrowckAnalyses<B::Domain, U::Domain, E::Domain>;
+
+    fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
+        BorrowckAnalyses {
+            borrows: self.borrows.analysis.bottom_value(body),
+            uninits: self.uninits.analysis.bottom_value(body),
+            ever_inits: self.ever_inits.analysis.bottom_value(body),
+        }
+    }
 
-            fn reconstruct_before_statement_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                stmt: &mir::Statement<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_before_statement_effect(&mut state.$field, stmt, loc); )*
-            }
+    fn reset_to_block_entry(&self, state: &mut Self::FlowState, block: BasicBlock) {
+        state.borrows.clone_from(&self.borrows.entry_set_for_block(block));
+        state.uninits.clone_from(&self.uninits.entry_set_for_block(block));
+        state.ever_inits.clone_from(&self.ever_inits.entry_set_for_block(block));
+    }
 
-            fn reconstruct_statement_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                stmt: &mir::Statement<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_statement_effect(&mut state.$field, stmt, loc); )*
-            }
+    fn reconstruct_before_statement_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        stmt: &mir::Statement<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_before_statement_effect(&mut state.borrows, stmt, loc);
+        self.uninits.analysis.apply_before_statement_effect(&mut state.uninits, stmt, loc);
+        self.ever_inits.analysis.apply_before_statement_effect(&mut state.ever_inits, stmt, loc);
+    }
 
-            fn reconstruct_before_terminator_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                term: &mir::Terminator<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_before_terminator_effect(&mut state.$field, term, loc); )*
-            }
+    fn reconstruct_statement_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        stmt: &mir::Statement<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_statement_effect(&mut state.borrows, stmt, loc);
+        self.uninits.analysis.apply_statement_effect(&mut state.uninits, stmt, loc);
+        self.ever_inits.analysis.apply_statement_effect(&mut state.ever_inits, stmt, loc);
+    }
 
-            fn reconstruct_terminator_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                term: &mir::Terminator<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_terminator_effect(&mut state.$field, term, loc); )*
-            }
-        }
-    )* }
-}
+    fn reconstruct_before_terminator_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        term: &mir::Terminator<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_before_terminator_effect(&mut state.borrows, term, loc);
+        self.uninits.analysis.apply_before_terminator_effect(&mut state.uninits, term, loc);
+        self.ever_inits.analysis.apply_before_terminator_effect(&mut state.ever_inits, term, loc);
+    }
 
-impl_visitable! {
-    BorrowckAnalyses { borrows: B, uninits: U, ever_inits: E }
+    fn reconstruct_terminator_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        term: &mir::Terminator<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_terminator_effect(&mut state.borrows, term, loc);
+        self.uninits.analysis.apply_terminator_effect(&mut state.uninits, term, loc);
+        self.ever_inits.analysis.apply_terminator_effect(&mut state.ever_inits, term, loc);
+    }
 }
 
 rustc_index::newtype_index! {
diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs
index 52cf87b676d..8b8a16bda99 100644
--- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs
@@ -84,8 +84,8 @@ pub trait ResultsVisitor<'mir, 'tcx, R> {
 
 /// Things that can be visited by a `ResultsVisitor`.
 ///
-/// This trait exists so that we can visit the results of multiple dataflow analyses simultaneously.
-/// DO NOT IMPLEMENT MANUALLY. Instead, use the `impl_visitable` macro below.
+/// This trait exists so that we can visit the results of one or more dataflow analyses
+/// simultaneously.
 pub trait ResultsVisitable<'tcx> {
     type Direction: Direction;
     type FlowState;
-- 
cgit 1.4.1-3-g733a5


From 55559d93e74d5cf48a041803597201640046b5bd Mon Sep 17 00:00:00 2001
From: León Orell Valerian Liehr <me@fmease.dev>
Date: Fri, 24 Nov 2023 09:53:10 +0100
Subject: Resolve assoc item bindings by namespace

If a const is expected, resolve a const.
If a type is expected, resolve a type.
Don't try to resolve a type first falling back to consts.
---
 compiler/rustc_hir_analysis/messages.ftl           |  31 ++-
 compiler/rustc_hir_analysis/src/astconv/bounds.rs  | 134 ++++--------
 compiler/rustc_hir_analysis/src/astconv/errors.rs  | 225 +++++++++++++--------
 compiler/rustc_hir_analysis/src/astconv/mod.rs     | 200 ++++++++----------
 compiler/rustc_hir_analysis/src/errors.rs          | 144 ++++++++++---
 .../associated-consts/assoc-const-eq-ambiguity.rs  |  19 ++
 .../assoc-const-eq-ambiguity.stderr                |  38 ++++
 .../ui/associated-consts/assoc-const-eq-missing.rs |  13 +-
 .../assoc-const-eq-missing.stderr                  |  20 +-
 .../assoc-const-eq-ty-alias-noninteracting.rs      |  21 ++
 .../associated-consts/assoc-const-ty-mismatch.rs   |  20 +-
 .../assoc-const-ty-mismatch.stderr                 |  36 ++--
 tests/ui/associated-consts/shadowed-const.rs       |   2 +-
 tests/ui/associated-consts/shadowed-const.stderr   |  12 +-
 tests/ui/associated-type-bounds/consts.rs          |   2 +-
 tests/ui/associated-type-bounds/consts.stderr      |  12 +-
 tests/ui/associated-type-bounds/issue-99828.rs     |   2 +-
 tests/ui/associated-type-bounds/issue-99828.stderr |  10 +-
 .../return-type-notation/missing.rs                |   2 +-
 .../return-type-notation/missing.stderr            |   5 +-
 .../super-method-bound-ambig.rs                    |   2 +-
 .../super-method-bound-ambig.stderr                |  13 +-
 .../ui/const-generics/assoc_const_eq_diagnostic.rs |   6 +-
 .../assoc_const_eq_diagnostic.stderr               |  28 ++-
 tests/ui/error-codes/E0221.stderr                  |   2 +-
 .../feature-gate-return_type_notation.cfg.stderr   |  13 +-
 .../feature-gate-return_type_notation.rs           |   2 +-
 27 files changed, 598 insertions(+), 416 deletions(-)
 create mode 100644 tests/ui/associated-consts/assoc-const-eq-ambiguity.rs
 create mode 100644 tests/ui/associated-consts/assoc-const-eq-ambiguity.stderr
 create mode 100644 tests/ui/associated-consts/assoc-const-eq-ty-alias-noninteracting.rs

diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl
index 8ab91bebcf6..139e1c0ac5f 100644
--- a/compiler/rustc_hir_analysis/messages.ftl
+++ b/compiler/rustc_hir_analysis/messages.ftl
@@ -1,8 +1,28 @@
+hir_analysis_ambiguous_assoc_item = ambiguous associated {$assoc_kind} `{$assoc_name}` in bounds of `{$ty_param_name}`
+    .label = ambiguous associated {$assoc_kind} `{$assoc_name}`
+
 hir_analysis_ambiguous_lifetime_bound =
     ambiguous lifetime bound, explicit lifetime bound required
 
-hir_analysis_assoc_bound_on_const = expected associated type, found {$descr}
-    .note = trait bounds not allowed on {$descr}
+hir_analysis_assoc_item_not_found = associated {$assoc_kind} `{$assoc_name}` not found for `{$ty_param_name}`
+
+hir_analysis_assoc_item_not_found_found_in_other_trait_label = there is {$identically_named ->
+        [true] an
+        *[false] a similarly named
+    } associated {$assoc_kind} `{$suggested_name}` in the trait `{$trait_name}`
+hir_analysis_assoc_item_not_found_label = associated {$assoc_kind} `{$assoc_name}` not found
+hir_analysis_assoc_item_not_found_other_sugg = `{$ty_param_name}` has the following associated {$assoc_kind}
+hir_analysis_assoc_item_not_found_similar_in_other_trait_sugg = change the associated {$assoc_kind} name to use `{$suggested_name}` from `{$trait_name}`
+hir_analysis_assoc_item_not_found_similar_in_other_trait_with_bound_sugg = and also change the associated {$assoc_kind} name
+hir_analysis_assoc_item_not_found_similar_sugg = there is an associated {$assoc_kind} with a similar name
+
+hir_analysis_assoc_kind_mismatch = expected {$expected}, found {$got}
+    .label = unexpected {$got}
+    .expected_because_label = expected a {$expected} because of this associated {$expected}
+    .note = the associated {$assoc_kind} is defined here
+    .bound_on_assoc_const_label = bounds are not allowed on associated constants
+
+hir_analysis_assoc_kind_mismatch_wrap_in_braces_sugg = consider adding braces here
 
 hir_analysis_assoc_type_binding_not_allowed =
     associated type bindings are not allowed here
@@ -280,10 +300,6 @@ hir_analysis_placeholder_not_allowed_item_signatures = the placeholder `_` is no
 
 hir_analysis_requires_note = the `{$trait_name}` impl for `{$ty}` requires that `{$error_predicate}`
 
-hir_analysis_return_type_notation_conflicting_bound =
-    ambiguous associated function `{$assoc_name}` for `{$ty_name}`
-    .note = `{$assoc_name}` is declared in two supertraits: `{$first_bound}` and `{$second_bound}`
-
 hir_analysis_return_type_notation_equality_bound =
     return type notation is not allowed to use type equality
 
@@ -294,9 +310,6 @@ hir_analysis_return_type_notation_illegal_param_type =
     return type notation is not allowed for functions that have type parameters
     .label = type parameter declared here
 
-hir_analysis_return_type_notation_missing_method =
-    cannot find associated function `{$assoc_name}` for `{$ty_name}`
-
 hir_analysis_return_type_notation_on_non_rpitit =
     return type notation used on function that is not `async` and does not return `impl Trait`
     .note = function returns `{$ty}`, which is not compatible with associated type return bounds
diff --git a/compiler/rustc_hir_analysis/src/astconv/bounds.rs b/compiler/rustc_hir_analysis/src/astconv/bounds.rs
index f6907019d6e..dfec3c5e829 100644
--- a/compiler/rustc_hir_analysis/src/astconv/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/bounds.rs
@@ -3,8 +3,7 @@ use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_lint_defs::Applicability;
-use rustc_middle::ty::{self as ty, Ty, TypeVisitableExt};
+use rustc_middle::ty::{self as ty, Ty};
 use rustc_span::symbol::Ident;
 use rustc_span::{ErrorGuaranteed, Span};
 use rustc_trait_selection::traits;
@@ -256,64 +255,49 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
 
         let tcx = self.tcx();
 
-        let return_type_notation =
-            binding.gen_args.parenthesized == hir::GenericArgsParentheses::ReturnTypeNotation;
-
-        let candidate = if return_type_notation {
-            if self.trait_defines_associated_item_named(
-                trait_ref.def_id(),
-                ty::AssocKind::Fn,
-                binding.item_name,
-            ) {
-                trait_ref
+        let assoc_kind =
+            if binding.gen_args.parenthesized == hir::GenericArgsParentheses::ReturnTypeNotation {
+                ty::AssocKind::Fn
+            } else if let ConvertedBindingKind::Equality(term) = binding.kind
+                && let ty::TermKind::Const(_) = term.node.unpack()
+            {
+                ty::AssocKind::Const
             } else {
-                self.one_bound_for_assoc_method(
-                    traits::supertraits(tcx, trait_ref),
-                    trait_ref.print_only_trait_path(),
-                    binding.item_name,
-                    path_span,
-                )?
-            }
-        } else if self.trait_defines_associated_item_named(
+                ty::AssocKind::Type
+            };
+
+        let candidate = if self.trait_defines_associated_item_named(
             trait_ref.def_id(),
-            ty::AssocKind::Type,
+            assoc_kind,
             binding.item_name,
         ) {
-            // Simple case: X is defined in the current trait.
+            // Simple case: The assoc item is defined in the current trait.
             trait_ref
         } else {
             // Otherwise, we have to walk through the supertraits to find
-            // those that do.
-            self.one_bound_for_assoc_type(
+            // one that does define it.
+            self.one_bound_for_assoc_item(
                 || traits::supertraits(tcx, trait_ref),
                 trait_ref.skip_binder().print_only_trait_name(),
                 None,
+                assoc_kind,
                 binding.item_name,
                 path_span,
-                match binding.kind {
-                    ConvertedBindingKind::Equality(term) => Some(term),
-                    _ => None,
-                },
+                Some(&binding),
             )?
         };
 
         let (assoc_ident, def_scope) =
             tcx.adjust_ident_and_get_scope(binding.item_name, candidate.def_id(), hir_ref_id);
 
-        // We have already adjusted the item name above, so compare with `ident.normalize_to_macros_2_0()` instead
-        // of calling `filter_by_name_and_kind`.
-        let find_item_of_kind = |kind| {
-            tcx.associated_items(candidate.def_id())
-                .filter_by_name_unhygienic(assoc_ident.name)
-                .find(|i| i.kind == kind && i.ident(tcx).normalize_to_macros_2_0() == assoc_ident)
-        };
-        let assoc_item = if return_type_notation {
-            find_item_of_kind(ty::AssocKind::Fn)
-        } else {
-            find_item_of_kind(ty::AssocKind::Type)
-                .or_else(|| find_item_of_kind(ty::AssocKind::Const))
-        }
-        .expect("missing associated type");
+        // We have already adjusted the item name above, so compare with `.normalize_to_macros_2_0()`
+        // instead of calling `filter_by_name_and_kind` which would needlessly normalize the
+        // `assoc_ident` again and again.
+        let assoc_item = tcx
+            .associated_items(candidate.def_id())
+            .filter_by_name_unhygienic(assoc_ident.name)
+            .find(|i| i.kind == assoc_kind && i.ident(tcx).normalize_to_macros_2_0() == assoc_ident)
+            .expect("missing associated item");
 
         if !assoc_item.visibility(tcx).is_accessible_from(def_scope, tcx) {
             tcx.sess
@@ -340,7 +324,7 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
                 .or_insert(binding.span);
         }
 
-        let projection_ty = if return_type_notation {
+        let projection_ty = if let ty::AssocKind::Fn = assoc_kind {
             let mut emitted_bad_param_err = false;
             // If we have an method return type bound, then we need to substitute
             // the method's early bound params with suitable late-bound params.
@@ -467,7 +451,7 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
                 let late_bound_in_trait_ref =
                     tcx.collect_constrained_late_bound_regions(&projection_ty);
                 let late_bound_in_ty =
-                    tcx.collect_referenced_late_bound_regions(&trait_ref.rebind(ty));
+                    tcx.collect_referenced_late_bound_regions(&trait_ref.rebind(ty.node));
                 debug!(?late_bound_in_trait_ref);
                 debug!(?late_bound_in_ty);
 
@@ -492,77 +476,27 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
             }
         }
 
-        let assoc_item_def_id = projection_ty.skip_binder().def_id;
-        let def_kind = tcx.def_kind(assoc_item_def_id);
         match binding.kind {
-            ConvertedBindingKind::Equality(..) if return_type_notation => {
+            ConvertedBindingKind::Equality(..) if let ty::AssocKind::Fn = assoc_kind => {
                 return Err(self.tcx().sess.emit_err(
                     crate::errors::ReturnTypeNotationEqualityBound { span: binding.span },
                 ));
             }
-            ConvertedBindingKind::Equality(mut term) => {
+            ConvertedBindingKind::Equality(term) => {
                 // "Desugar" a constraint like `T: Iterator<Item = u32>` this to
                 // the "projection predicate" for:
                 //
                 // `<T as Iterator>::Item = u32`
-                match (def_kind, term.unpack()) {
-                    (DefKind::AssocTy, ty::TermKind::Ty(_))
-                    | (DefKind::AssocConst, ty::TermKind::Const(_)) => (),
-                    (_, _) => {
-                        let got = if let Some(_) = term.ty() { "type" } else { "constant" };
-                        let expected = tcx.def_descr(assoc_item_def_id);
-                        let mut err = tcx.sess.struct_span_err(
-                            binding.span,
-                            format!("expected {expected} bound, found {got}"),
-                        );
-                        err.span_note(
-                            tcx.def_span(assoc_item_def_id),
-                            format!("{expected} defined here"),
-                        );
-
-                        if let DefKind::AssocConst = def_kind
-                            && let Some(t) = term.ty()
-                            && (t.is_enum() || t.references_error())
-                            && tcx.features().associated_const_equality
-                        {
-                            err.span_suggestion(
-                                binding.span,
-                                "if equating a const, try wrapping with braces",
-                                format!("{} = {{ const }}", binding.item_name),
-                                Applicability::HasPlaceholders,
-                            );
-                        }
-                        let reported = err.emit();
-                        term = match def_kind {
-                            DefKind::AssocTy => Ty::new_error(tcx, reported).into(),
-                            DefKind::AssocConst => ty::Const::new_error(
-                                tcx,
-                                reported,
-                                tcx.type_of(assoc_item_def_id)
-                                    .instantiate(tcx, projection_ty.skip_binder().args),
-                            )
-                            .into(),
-                            _ => unreachable!(),
-                        };
-                    }
-                }
                 bounds.push_projection_bound(
                     tcx,
-                    projection_ty
-                        .map_bound(|projection_ty| ty::ProjectionPredicate { projection_ty, term }),
+                    projection_ty.map_bound(|projection_ty| ty::ProjectionPredicate {
+                        projection_ty,
+                        term: term.node,
+                    }),
                     binding.span,
                 );
             }
             ConvertedBindingKind::Constraint(ast_bounds) => {
-                match def_kind {
-                    DefKind::AssocTy => {}
-                    _ => {
-                        return Err(tcx.sess.emit_err(errors::AssocBoundOnConst {
-                            span: assoc_ident.span,
-                            descr: tcx.def_descr(assoc_item_def_id),
-                        }));
-                    }
-                }
                 // "Desugar" a constraint like `T: Iterator<Item: Debug>` to
                 //
                 // `<T as Iterator>::Item: Debug`
diff --git a/compiler/rustc_hir_analysis/src/astconv/errors.rs b/compiler/rustc_hir_analysis/src/astconv/errors.rs
index eb12ec7a098..13ad9a453b2 100644
--- a/compiler/rustc_hir_analysis/src/astconv/errors.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/errors.rs
@@ -1,15 +1,16 @@
-use crate::astconv::AstConv;
+use crate::astconv::{AstConv, ConvertedBindingKind};
 use crate::errors::{
-    AssocTypeBindingNotAllowed, ManualImplementation, MissingTypeParams,
+    self, AssocTypeBindingNotAllowed, ManualImplementation, MissingTypeParams,
     ParenthesizedFnTraitExpansion,
 };
+use crate::fluent_generated as fluent;
 use crate::traits::error_reporting::report_object_safety_error;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_errors::{pluralize, struct_span_err, Applicability, Diagnostic, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_infer::traits::FulfillmentError;
-use rustc_middle::ty::{self, suggest_constraining_type_param, AssocItem, AssocKind, Ty, TyCtxt};
+use rustc_middle::ty::{self, suggest_constraining_type_param, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::parse::feature_err;
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::symbol::{sym, Ident};
@@ -97,83 +98,88 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         }
     }
 
-    pub(crate) fn complain_about_assoc_type_not_found<I>(
+    pub(super) fn complain_about_assoc_item_not_found<I>(
         &self,
         all_candidates: impl Fn() -> I,
         ty_param_name: &str,
         ty_param_def_id: Option<LocalDefId>,
+        assoc_kind: ty::AssocKind,
         assoc_name: Ident,
         span: Span,
+        binding: Option<&super::ConvertedBinding<'_, 'tcx>>,
     ) -> ErrorGuaranteed
     where
         I: Iterator<Item = ty::PolyTraitRef<'tcx>>,
     {
+        let tcx = self.tcx();
+
+        // First and foremost, provide a more user-friendly & “intuitive” error on kind mismatches.
+        if let Some(assoc_item) = all_candidates().find_map(|r| {
+            tcx.associated_items(r.def_id())
+                .filter_by_name_unhygienic(assoc_name.name)
+                .find(|item| tcx.hygienic_eq(assoc_name, item.ident(tcx), r.def_id()))
+        }) {
+            return self.complain_about_assoc_kind_mismatch(
+                assoc_item, assoc_kind, assoc_name, span, binding,
+            );
+        }
+
+        let assoc_kind_str = super::assoc_kind_str(assoc_kind);
+
         // The fallback span is needed because `assoc_name` might be an `Fn()`'s `Output` without a
         // valid span, so we point at the whole path segment instead.
         let is_dummy = assoc_name.span == DUMMY_SP;
 
-        let mut err = struct_span_err!(
-            self.tcx().sess,
-            if is_dummy { span } else { assoc_name.span },
-            E0220,
-            "associated type `{}` not found for `{}`",
+        let mut err = errors::AssocItemNotFound {
+            span: if is_dummy { span } else { assoc_name.span },
             assoc_name,
-            ty_param_name
-        );
+            assoc_kind: assoc_kind_str,
+            ty_param_name,
+            label: None,
+            sugg: None,
+        };
 
         if is_dummy {
-            err.span_label(span, format!("associated type `{assoc_name}` not found"));
-            return err.emit();
+            err.label = Some(errors::AssocItemNotFoundLabel::NotFound { span });
+            return tcx.sess.emit_err(err);
         }
 
         let all_candidate_names: Vec<_> = all_candidates()
-            .flat_map(|r| self.tcx().associated_items(r.def_id()).in_definition_order())
+            .flat_map(|r| tcx.associated_items(r.def_id()).in_definition_order())
             .filter_map(|item| {
-                if !item.is_impl_trait_in_trait() && item.kind == ty::AssocKind::Type {
-                    Some(item.name)
-                } else {
-                    None
-                }
+                (!item.is_impl_trait_in_trait() && item.kind == assoc_kind).then_some(item.name)
             })
             .collect();
 
         if let Some(suggested_name) =
             find_best_match_for_name(&all_candidate_names, assoc_name.name, None)
         {
-            err.span_suggestion(
-                assoc_name.span,
-                "there is an associated type with a similar name",
+            err.sugg = Some(errors::AssocItemNotFoundSugg::Similar {
+                span: assoc_name.span,
+                assoc_kind: assoc_kind_str,
                 suggested_name,
-                Applicability::MaybeIncorrect,
-            );
-            return err.emit();
+            });
+            return tcx.sess.emit_err(err);
         }
 
         // If we didn't find a good item in the supertraits (or couldn't get
         // the supertraits), like in ItemCtxt, then look more generally from
         // all visible traits. If there's one clear winner, just suggest that.
 
-        let visible_traits: Vec<_> = self
-            .tcx()
+        let visible_traits: Vec<_> = tcx
             .all_traits()
             .filter(|trait_def_id| {
-                let viz = self.tcx().visibility(*trait_def_id);
+                let viz = tcx.visibility(*trait_def_id);
                 let def_id = self.item_def_id();
-                viz.is_accessible_from(def_id, self.tcx())
+                viz.is_accessible_from(def_id, tcx)
             })
             .collect();
 
         let wider_candidate_names: Vec<_> = visible_traits
             .iter()
-            .flat_map(|trait_def_id| {
-                self.tcx().associated_items(*trait_def_id).in_definition_order()
-            })
+            .flat_map(|trait_def_id| tcx.associated_items(*trait_def_id).in_definition_order())
             .filter_map(|item| {
-                if !item.is_impl_trait_in_trait() && item.kind == ty::AssocKind::Type {
-                    Some(item.name)
-                } else {
-                    None
-                }
+                (!item.is_impl_trait_in_trait() && item.kind == assoc_kind).then_some(item.name)
             })
             .collect();
 
@@ -182,52 +188,51 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         {
             if let [best_trait] = visible_traits
                 .iter()
+                .copied()
                 .filter(|trait_def_id| {
-                    self.tcx()
-                        .associated_items(*trait_def_id)
+                    tcx.associated_items(trait_def_id)
                         .filter_by_name_unhygienic(suggested_name)
-                        .any(|item| item.kind == ty::AssocKind::Type)
+                        .any(|item| item.kind == assoc_kind)
                 })
                 .collect::<Vec<_>>()[..]
             {
-                let trait_name = self.tcx().def_path_str(*best_trait);
-                let an = if suggested_name != assoc_name.name { "a similarly named" } else { "an" };
-                err.span_label(
-                    assoc_name.span,
-                    format!(
-                        "there is {an} associated type `{suggested_name}` in the \
-                         trait `{trait_name}`",
-                    ),
-                );
-                let hir = self.tcx().hir();
+                let trait_name = tcx.def_path_str(best_trait);
+                err.label = Some(errors::AssocItemNotFoundLabel::FoundInOtherTrait {
+                    span: assoc_name.span,
+                    assoc_kind: assoc_kind_str,
+                    trait_name: &trait_name,
+                    suggested_name,
+                    identically_named: suggested_name == assoc_name.name,
+                });
+                let hir = tcx.hir();
                 if let Some(def_id) = ty_param_def_id
-                    && let parent = hir.get_parent_item(self.tcx().local_def_id_to_hir_id(def_id))
+                    && let parent = hir.get_parent_item(tcx.local_def_id_to_hir_id(def_id))
                     && let Some(generics) = hir.get_generics(parent.def_id)
                 {
                     if generics.bounds_for_param(def_id).flat_map(|pred| pred.bounds.iter()).any(
                         |b| match b {
                             hir::GenericBound::Trait(t, ..) => {
-                                t.trait_ref.trait_def_id().as_ref() == Some(best_trait)
+                                t.trait_ref.trait_def_id() == Some(best_trait)
                             }
                             _ => false,
                         },
                     ) {
                         // The type param already has a bound for `trait_name`, we just need to
-                        // change the associated type.
-                        err.span_suggestion_verbose(
-                            assoc_name.span,
-                            format!(
-                                "change the associated type name to use `{suggested_name}` from \
-                                 `{trait_name}`",
-                            ),
-                            suggested_name.to_string(),
-                            Applicability::MaybeIncorrect,
-                        );
-                    } else if suggest_constraining_type_param(
-                        self.tcx(),
+                        // change the associated item.
+                        err.sugg = Some(errors::AssocItemNotFoundSugg::SimilarInOtherTrait {
+                            span: assoc_name.span,
+                            assoc_kind: assoc_kind_str,
+                            suggested_name,
+                        });
+                        return tcx.sess.emit_err(err);
+                    }
+
+                    let mut err = tcx.sess.create_err(err);
+                    if suggest_constraining_type_param(
+                        tcx,
                         generics,
                         &mut err,
-                        ty_param_name,
+                        &ty_param_name,
                         &trait_name,
                         None,
                         None,
@@ -237,39 +242,101 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         // was also not an exact match, so we also suggest changing it.
                         err.span_suggestion_verbose(
                             assoc_name.span,
-                            "and also change the associated type name",
+                            fluent::hir_analysis_assoc_item_not_found_similar_in_other_trait_with_bound_sugg,
                             suggested_name.to_string(),
                             Applicability::MaybeIncorrect,
                         );
                     }
+                    return err.emit();
                 }
-                return err.emit();
+                return tcx.sess.emit_err(err);
             }
         }
 
         // If we still couldn't find any associated type, and only one associated type exists,
         // suggests using it.
-
-        if all_candidate_names.len() == 1 {
+        if let [candidate_name] = all_candidate_names.as_slice() {
             // this should still compile, except on `#![feature(associated_type_defaults)]`
             // where it could suggests `type A = Self::A`, thus recursing infinitely
-            let applicability = if self.tcx().features().associated_type_defaults {
+            let applicability = if tcx.features().associated_type_defaults {
                 Applicability::Unspecified
             } else {
                 Applicability::MaybeIncorrect
             };
 
-            err.span_suggestion(
-                assoc_name.span,
-                format!("`{ty_param_name}` has the following associated type"),
-                all_candidate_names.first().unwrap().to_string(),
+            err.sugg = Some(errors::AssocItemNotFoundSugg::Other {
+                span: assoc_name.span,
                 applicability,
-            );
+                ty_param_name,
+                assoc_kind: assoc_kind_str,
+                suggested_name: *candidate_name,
+            });
         } else {
-            err.span_label(assoc_name.span, format!("associated type `{assoc_name}` not found"));
+            err.label = Some(errors::AssocItemNotFoundLabel::NotFound { span: assoc_name.span });
         }
 
-        err.emit()
+        tcx.sess.emit_err(err)
+    }
+
+    fn complain_about_assoc_kind_mismatch(
+        &self,
+        assoc_item: &ty::AssocItem,
+        assoc_kind: ty::AssocKind,
+        ident: Ident,
+        span: Span,
+        binding: Option<&super::ConvertedBinding<'_, 'tcx>>,
+    ) -> ErrorGuaranteed {
+        let tcx = self.tcx();
+
+        let bound_on_assoc_const_label = if let ty::AssocKind::Const = assoc_item.kind
+            && let Some(binding) = binding
+            && let ConvertedBindingKind::Constraint(_) = binding.kind
+        {
+            let lo = if binding.gen_args.span_ext.is_dummy() {
+                ident.span
+            } else {
+                binding.gen_args.span_ext
+            };
+            Some(lo.between(span.shrink_to_hi()))
+        } else {
+            None
+        };
+
+        // FIXME(associated_const_equality): This has quite a few false positives and negatives.
+        let wrap_in_braces_sugg = if let Some(binding) = binding
+            && let ConvertedBindingKind::Equality(term) = binding.kind
+            && let ty::TermKind::Ty(ty) = term.node.unpack()
+            && (ty.is_enum() || ty.references_error())
+            && tcx.features().associated_const_equality
+        {
+            Some(errors::AssocKindMismatchWrapInBracesSugg {
+                lo: term.span.shrink_to_lo(),
+                hi: term.span.shrink_to_hi(),
+            })
+        } else {
+            None
+        };
+
+        // For equality bounds, we want to blame the term (RHS) instead of the item (LHS) since
+        // one can argue that that's more “untuitive” to the user.
+        let (span, expected_because_label, expected, got) = if let Some(binding) = binding
+            && let ConvertedBindingKind::Equality(term) = binding.kind
+        {
+            (term.span, Some(ident.span), assoc_item.kind, assoc_kind)
+        } else {
+            (ident.span, None, assoc_kind, assoc_item.kind)
+        };
+
+        tcx.sess.emit_err(errors::AssocKindMismatch {
+            span,
+            expected: super::assoc_kind_str(expected),
+            got: super::assoc_kind_str(got),
+            expected_because_label,
+            assoc_kind: super::assoc_kind_str(assoc_item.kind),
+            def_span: tcx.def_span(assoc_item.def_id),
+            bound_on_assoc_const_label,
+            wrap_in_braces_sugg,
+        })
     }
 
     pub(crate) fn complain_about_ambiguous_inherent_assoc_type(
@@ -598,7 +665,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     let assoc_item = tcx.associated_items(trait_def).find_by_name_and_kind(
                         tcx,
                         ident,
-                        AssocKind::Type,
+                        ty::AssocKind::Type,
                         trait_def,
                     );
 
@@ -606,7 +673,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 }))
             })
             .flatten()
-            .collect::<FxHashMap<Symbol, &AssocItem>>();
+            .collect::<FxHashMap<Symbol, &ty::AssocItem>>();
 
         let mut names = names
             .into_iter()
diff --git a/compiler/rustc_hir_analysis/src/astconv/mod.rs b/compiler/rustc_hir_analysis/src/astconv/mod.rs
index 0f06407f445..20d36a1b069 100644
--- a/compiler/rustc_hir_analysis/src/astconv/mod.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/mod.rs
@@ -18,8 +18,8 @@ use crate::require_c_abi_if_c_variadic;
 use rustc_ast::TraitObjectSyntax;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::{
-    struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, FatalError,
-    MultiSpan,
+    error_code, struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
+    FatalError, MultiSpan,
 };
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Namespace, Res};
@@ -36,6 +36,7 @@ use rustc_middle::ty::{
 };
 use rustc_session::lint::builtin::AMBIGUOUS_ASSOCIATED_ITEMS;
 use rustc_span::edit_distance::find_best_match_for_name;
+use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::{sym, BytePos, Span, DUMMY_SP};
 use rustc_target::spec::abi;
@@ -162,7 +163,7 @@ struct ConvertedBinding<'a, 'tcx> {
 
 #[derive(Debug)]
 enum ConvertedBindingKind<'a, 'tcx> {
-    Equality(ty::Term<'tcx>),
+    Equality(Spanned<ty::Term<'tcx>>),
     Constraint(&'a [hir::GenericBound<'a>]),
 }
 
@@ -595,12 +596,14 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             .map(|binding| {
                 let kind = match &binding.kind {
                     hir::TypeBindingKind::Equality { term } => match term {
-                        hir::Term::Ty(ty) => {
-                            ConvertedBindingKind::Equality(self.ast_ty_to_ty(ty).into())
-                        }
+                        hir::Term::Ty(ty) => ConvertedBindingKind::Equality(respan(
+                            ty.span,
+                            self.ast_ty_to_ty(ty).into(),
+                        )),
                         hir::Term::Const(c) => {
+                            let span = self.tcx().def_span(c.def_id);
                             let c = Const::from_anon_const(self.tcx(), c.def_id);
-                            ConvertedBindingKind::Equality(c.into())
+                            ConvertedBindingKind::Equality(respan(span, c.into()))
                         }
                     },
                     hir::TypeBindingKind::Constraint { bounds } => {
@@ -1056,7 +1059,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         debug!("find_bound_for_assoc_item: predicates={:#?}", predicates);
 
         let param_name = tcx.hir().ty_param_name(ty_param_def_id);
-        self.one_bound_for_assoc_type(
+        self.one_bound_for_assoc_item(
             || {
                 traits::transitive_bounds_that_define_assoc_item(
                     tcx,
@@ -1068,6 +1071,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             },
             param_name,
             Some(ty_param_def_id),
+            ty::AssocKind::Type,
             assoc_name,
             span,
             None,
@@ -1076,48 +1080,45 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
     // Checks that `bounds` contains exactly one element and reports appropriate
     // errors otherwise.
-    #[instrument(level = "debug", skip(self, all_candidates, ty_param_name, is_equality), ret)]
-    fn one_bound_for_assoc_type<I>(
+    #[instrument(level = "debug", skip(self, all_candidates, ty_param_name, binding), ret)]
+    fn one_bound_for_assoc_item<I>(
         &self,
         all_candidates: impl Fn() -> I,
         ty_param_name: impl Display,
         ty_param_def_id: Option<LocalDefId>,
+        assoc_kind: ty::AssocKind,
         assoc_name: Ident,
         span: Span,
-        is_equality: Option<ty::Term<'tcx>>,
+        binding: Option<&ConvertedBinding<'_, 'tcx>>,
     ) -> Result<ty::PolyTraitRef<'tcx>, ErrorGuaranteed>
     where
         I: Iterator<Item = ty::PolyTraitRef<'tcx>>,
     {
+        let tcx = self.tcx();
+
         let mut matching_candidates = all_candidates().filter(|r| {
-            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Type, assoc_name)
-        });
-        let mut const_candidates = all_candidates().filter(|r| {
-            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Const, assoc_name)
+            self.trait_defines_associated_item_named(r.def_id(), assoc_kind, assoc_name)
         });
 
-        let (mut bound, mut next_cand) = match (matching_candidates.next(), const_candidates.next())
-        {
-            (Some(bound), _) => (bound, matching_candidates.next()),
-            (None, Some(bound)) => (bound, const_candidates.next()),
-            (None, None) => {
-                let reported = self.complain_about_assoc_type_not_found(
-                    all_candidates,
-                    &ty_param_name.to_string(),
-                    ty_param_def_id,
-                    assoc_name,
-                    span,
-                );
-                return Err(reported);
-            }
+        let Some(mut bound) = matching_candidates.next() else {
+            let reported = self.complain_about_assoc_item_not_found(
+                all_candidates,
+                &ty_param_name.to_string(),
+                ty_param_def_id,
+                assoc_kind,
+                assoc_name,
+                span,
+                binding,
+            );
+            return Err(reported);
         };
         debug!(?bound);
 
         // look for a candidate that is not the same as our first bound, disregarding
         // whether the bound is const.
+        let mut next_cand = matching_candidates.next();
         while let Some(mut bound2) = next_cand {
             debug!(?bound2);
-            let tcx = self.tcx();
             if bound2.bound_vars() != bound.bound_vars() {
                 break;
             }
@@ -1138,7 +1139,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 .map(|(n, arg)| if host_index == n { tcx.consts.true_.into() } else { arg });
 
             if unconsted_args.eq(bound2.skip_binder().args.iter()) {
-                next_cand = matching_candidates.next().or_else(|| const_candidates.next());
+                next_cand = matching_candidates.next();
             } else {
                 break;
             }
@@ -1147,48 +1148,53 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         if let Some(bound2) = next_cand {
             debug!(?bound2);
 
-            let bounds = IntoIterator::into_iter([bound, bound2]).chain(matching_candidates);
-            let mut err = if is_equality.is_some() {
-                // More specific Error Index entry.
-                struct_span_err!(
-                    self.tcx().sess,
-                    span,
-                    E0222,
-                    "ambiguous associated type `{}` in bounds of `{}`",
-                    assoc_name,
-                    ty_param_name
-                )
-            } else {
-                struct_span_err!(
-                    self.tcx().sess,
-                    span,
-                    E0221,
-                    "ambiguous associated type `{}` in bounds of `{}`",
-                    assoc_name,
-                    ty_param_name
-                )
-            };
-            err.span_label(span, format!("ambiguous associated type `{assoc_name}`"));
+            let assoc_kind_str = assoc_kind_str(assoc_kind);
+            let ty_param_name = &ty_param_name.to_string();
+            let mut err = tcx.sess.create_err(crate::errors::AmbiguousAssocItem {
+                span,
+                assoc_kind: assoc_kind_str,
+                assoc_name,
+                ty_param_name,
+            });
+            // Provide a more specific error code index entry for equality bindings.
+            err.code(
+                if let Some(binding) = binding
+                    && let ConvertedBindingKind::Equality(_) = binding.kind
+                {
+                    error_code!(E0222)
+                } else {
+                    error_code!(E0221)
+                },
+            );
 
+            // FIXME(#97583): Resugar equality bounds to type/const bindings.
+            // FIXME: Turn this into a structured, translateable & more actionable suggestion.
             let mut where_bounds = vec![];
-            for bound in bounds {
+            for bound in [bound, bound2].into_iter().chain(matching_candidates) {
                 let bound_id = bound.def_id();
-                let bound_span = self
-                    .tcx()
+                let bound_span = tcx
                     .associated_items(bound_id)
-                    .find_by_name_and_kind(self.tcx(), assoc_name, ty::AssocKind::Type, bound_id)
-                    .and_then(|item| self.tcx().hir().span_if_local(item.def_id));
+                    .find_by_name_and_kind(tcx, assoc_name, assoc_kind, bound_id)
+                    .and_then(|item| tcx.hir().span_if_local(item.def_id));
 
                 if let Some(bound_span) = bound_span {
                     err.span_label(
                         bound_span,
                         format!("ambiguous `{assoc_name}` from `{}`", bound.print_trait_sugared(),),
                     );
-                    if let Some(constraint) = &is_equality {
-                        where_bounds.push(format!(
-                            "        T: {trait}::{assoc_name} = {constraint}",
-                            trait=bound.print_only_trait_path(),
-                        ));
+                    if let Some(binding) = binding {
+                        match binding.kind {
+                            ConvertedBindingKind::Equality(term) => {
+                                // FIXME(#97583): This isn't syntactically well-formed!
+                                where_bounds.push(format!(
+                                    "        T: {trait}::{assoc_name} = {term}",
+                                    trait = bound.print_only_trait_path(),
+                                    term = term.node,
+                                ));
+                            }
+                            // FIXME: Provide a suggestion.
+                            ConvertedBindingKind::Constraint(_bounds) => {}
+                        }
                     } else {
                         err.span_suggestion_verbose(
                             span.with_hi(assoc_name.span.lo()),
@@ -1199,7 +1205,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     }
                 } else {
                     err.note(format!(
-                        "associated type `{ty_param_name}` could derive from `{}`",
+                        "associated {assoc_kind_str} `{assoc_name}` could derive from `{}`",
                         bound.print_only_trait_path(),
                     ));
                 }
@@ -1220,46 +1226,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         Ok(bound)
     }
 
-    #[instrument(level = "debug", skip(self, all_candidates, ty_name), ret)]
-    fn one_bound_for_assoc_method(
-        &self,
-        all_candidates: impl Iterator<Item = ty::PolyTraitRef<'tcx>>,
-        ty_name: impl Display,
-        assoc_name: Ident,
-        span: Span,
-    ) -> Result<ty::PolyTraitRef<'tcx>, ErrorGuaranteed> {
-        let mut matching_candidates = all_candidates.filter(|r| {
-            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Fn, assoc_name)
-        });
-
-        let candidate = match matching_candidates.next() {
-            Some(candidate) => candidate,
-            None => {
-                return Err(self.tcx().sess.emit_err(
-                    crate::errors::ReturnTypeNotationMissingMethod {
-                        span,
-                        ty_name: ty_name.to_string(),
-                        assoc_name: assoc_name.name,
-                    },
-                ));
-            }
-        };
-
-        if let Some(conflicting_candidate) = matching_candidates.next() {
-            return Err(self.tcx().sess.emit_err(
-                crate::errors::ReturnTypeNotationConflictingBound {
-                    span,
-                    ty_name: ty_name.to_string(),
-                    assoc_name: assoc_name.name,
-                    first_bound: candidate.print_only_trait_path(),
-                    second_bound: conflicting_candidate.print_only_trait_path(),
-                },
-            ));
-        }
-
-        Ok(candidate)
-    }
-
     // Create a type from a path to an associated type or to an enum variant.
     // For a path `A::B::C::D`, `qself_ty` and `qself_def` are the type and def for `A::B::C`
     // and item_segment is the path segment for `D`. We return a type and a def for
@@ -1421,7 +1387,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     return Err(guar);
                 };
 
-                self.one_bound_for_assoc_type(
+                self.one_bound_for_assoc_item(
                     || {
                         traits::supertraits(
                             tcx,
@@ -1430,6 +1396,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     },
                     kw::SelfUpper,
                     None,
+                    ty::AssocKind::Type,
                     assoc_ident,
                     span,
                     None,
@@ -1510,15 +1477,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         };
 
         let trait_did = bound.def_id();
-        let Some(assoc_ty_did) = self.lookup_assoc_ty(assoc_ident, hir_ref_id, span, trait_did)
-        else {
-            // Assume that if it's not matched, there must be a const defined with the same name
-            // but it was used in a type position.
-            let msg = format!("found associated const `{assoc_ident}` when type was expected");
-            let guar = tcx.sess.struct_span_err(span, msg).emit();
-            return Err(guar);
-        };
-
+        let assoc_ty_did = self.lookup_assoc_ty(assoc_ident, hir_ref_id, span, trait_did).unwrap();
         let ty = self.projected_ty_from_poly_trait_ref(span, assoc_ty_did, assoc_segment, bound);
 
         if let Some(variant_def_id) = variant_resolution {
@@ -1731,8 +1690,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         let tcx = self.tcx();
         let (ident, def_scope) = tcx.adjust_ident_and_get_scope(name, scope, block);
 
-        // We have already adjusted the item name above, so compare with `ident.normalize_to_macros_2_0()` instead
-        // of calling `find_by_name_and_kind`.
+        // We have already adjusted the item name above, so compare with `.normalize_to_macros_2_0()`
+        // instead of calling `filter_by_name_and_kind` which would needlessly normalize the
+        // `ident` again and again.
         let item = tcx.associated_items(scope).in_definition_order().find(|i| {
             i.kind.namespace() == Namespace::TypeNS
                 && i.ident(tcx).normalize_to_macros_2_0() == ident
@@ -2858,3 +2818,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         Some(r)
     }
 }
+
+fn assoc_kind_str(kind: ty::AssocKind) -> &'static str {
+    match kind {
+        ty::AssocKind::Fn => "function",
+        ty::AssocKind::Const => "constant",
+        ty::AssocKind::Type => "type",
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs
index dd83b5b6f2c..a4772293697 100644
--- a/compiler/rustc_hir_analysis/src/errors.rs
+++ b/compiler/rustc_hir_analysis/src/errors.rs
@@ -6,9 +6,121 @@ use rustc_errors::{
     MultiSpan,
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
-use rustc_middle::ty::{self, print::TraitRefPrintOnlyTraitPath, Ty};
+use rustc_middle::ty::Ty;
 use rustc_span::{symbol::Ident, Span, Symbol};
 
+#[derive(Diagnostic)]
+#[diag(hir_analysis_ambiguous_assoc_item)]
+pub struct AmbiguousAssocItem<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub assoc_kind: &'static str,
+    pub assoc_name: Ident,
+    pub ty_param_name: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_assoc_kind_mismatch)]
+pub struct AssocKindMismatch {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub expected: &'static str,
+    pub got: &'static str,
+    #[label(hir_analysis_expected_because_label)]
+    pub expected_because_label: Option<Span>,
+    pub assoc_kind: &'static str,
+    #[note]
+    pub def_span: Span,
+    #[label(hir_analysis_bound_on_assoc_const_label)]
+    pub bound_on_assoc_const_label: Option<Span>,
+    #[subdiagnostic]
+    pub wrap_in_braces_sugg: Option<AssocKindMismatchWrapInBracesSugg>,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    hir_analysis_assoc_kind_mismatch_wrap_in_braces_sugg,
+    applicability = "maybe-incorrect"
+)]
+pub struct AssocKindMismatchWrapInBracesSugg {
+    #[suggestion_part(code = "{{ ")]
+    pub lo: Span,
+    #[suggestion_part(code = " }}")]
+    pub hi: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_assoc_item_not_found, code = "E0220")]
+pub struct AssocItemNotFound<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub assoc_name: Ident,
+    pub assoc_kind: &'static str,
+    pub ty_param_name: &'a str,
+    #[subdiagnostic]
+    pub label: Option<AssocItemNotFoundLabel<'a>>,
+    #[subdiagnostic]
+    pub sugg: Option<AssocItemNotFoundSugg<'a>>,
+}
+
+#[derive(Subdiagnostic)]
+pub enum AssocItemNotFoundLabel<'a> {
+    #[label(hir_analysis_assoc_item_not_found_label)]
+    NotFound {
+        #[primary_span]
+        span: Span,
+    },
+    #[label(hir_analysis_assoc_item_not_found_found_in_other_trait_label)]
+    FoundInOtherTrait {
+        #[primary_span]
+        span: Span,
+        assoc_kind: &'static str,
+        trait_name: &'a str,
+        suggested_name: Symbol,
+        identically_named: bool,
+    },
+}
+
+#[derive(Subdiagnostic)]
+
+pub enum AssocItemNotFoundSugg<'a> {
+    #[suggestion(
+        hir_analysis_assoc_item_not_found_similar_sugg,
+        code = "{suggested_name}",
+        applicability = "maybe-incorrect"
+    )]
+    Similar {
+        #[primary_span]
+        span: Span,
+        assoc_kind: &'static str,
+        suggested_name: Symbol,
+    },
+    #[suggestion(
+        hir_analysis_assoc_item_not_found_similar_in_other_trait_sugg,
+        code = "{suggested_name}",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    SimilarInOtherTrait {
+        #[primary_span]
+        span: Span,
+        assoc_kind: &'static str,
+        suggested_name: Symbol,
+    },
+    #[suggestion(hir_analysis_assoc_item_not_found_other_sugg, code = "{suggested_name}")]
+    Other {
+        #[primary_span]
+        span: Span,
+        #[applicability]
+        applicability: Applicability,
+        ty_param_name: &'a str,
+        assoc_kind: &'static str,
+        suggested_name: Symbol,
+    },
+}
+
 #[derive(Diagnostic)]
 #[diag(hir_analysis_unrecognized_atomic_operation, code = "E0092")]
 pub struct UnrecognizedAtomicOperation<'a> {
@@ -537,27 +649,6 @@ pub(crate) struct ReturnTypeNotationEqualityBound {
     pub span: Span,
 }
 
-#[derive(Diagnostic)]
-#[diag(hir_analysis_return_type_notation_missing_method)]
-pub(crate) struct ReturnTypeNotationMissingMethod {
-    #[primary_span]
-    pub span: Span,
-    pub ty_name: String,
-    pub assoc_name: Symbol,
-}
-
-#[derive(Diagnostic)]
-#[diag(hir_analysis_return_type_notation_conflicting_bound)]
-#[note]
-pub(crate) struct ReturnTypeNotationConflictingBound<'tcx> {
-    #[primary_span]
-    pub span: Span,
-    pub ty_name: String,
-    pub assoc_name: Symbol,
-    pub first_bound: ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
-    pub second_bound: ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
-}
-
 #[derive(Diagnostic)]
 #[diag(hir_analysis_placeholder_not_allowed_item_signatures, code = "E0121")]
 pub(crate) struct PlaceholderNotAllowedItemSignatures {
@@ -954,15 +1045,6 @@ pub(crate) struct ReturnPositionImplTraitInTraitRefined<'tcx> {
     pub return_ty: Ty<'tcx>,
 }
 
-#[derive(Diagnostic)]
-#[diag(hir_analysis_assoc_bound_on_const)]
-#[note]
-pub struct AssocBoundOnConst {
-    #[primary_span]
-    pub span: Span,
-    pub descr: &'static str,
-}
-
 #[derive(Diagnostic)]
 #[diag(hir_analysis_inherent_ty_outside, code = "E0390")]
 #[help]
diff --git a/tests/ui/associated-consts/assoc-const-eq-ambiguity.rs b/tests/ui/associated-consts/assoc-const-eq-ambiguity.rs
new file mode 100644
index 00000000000..ac085864ff0
--- /dev/null
+++ b/tests/ui/associated-consts/assoc-const-eq-ambiguity.rs
@@ -0,0 +1,19 @@
+// We used to say "ambiguous associated type" on ambiguous associated consts.
+// Ensure that we now use the correct label.
+
+#![feature(associated_const_equality)]
+
+trait Trait0: Parent0<i32> + Parent0<u32> {}
+trait Parent0<T> { const K: (); }
+
+fn take0(_: impl Trait0<K = { () }>) {}
+//~^ ERROR ambiguous associated constant `K` in bounds of `Trait0`
+
+trait Trait1: Parent1 + Parent2 {}
+trait Parent1 { const C: i32; }
+trait Parent2 { const C: &'static str; }
+
+fn take1(_: impl Trait1<C = "?">) {}
+//~^ ERROR ambiguous associated constant `C` in bounds of `Trait1`
+
+fn main() {}
diff --git a/tests/ui/associated-consts/assoc-const-eq-ambiguity.stderr b/tests/ui/associated-consts/assoc-const-eq-ambiguity.stderr
new file mode 100644
index 00000000000..ba3a8701316
--- /dev/null
+++ b/tests/ui/associated-consts/assoc-const-eq-ambiguity.stderr
@@ -0,0 +1,38 @@
+error[E0222]: ambiguous associated constant `K` in bounds of `Trait0`
+  --> $DIR/assoc-const-eq-ambiguity.rs:9:25
+   |
+LL | trait Parent0<T> { const K: (); }
+   |                    -----------
+   |                    |
+   |                    ambiguous `K` from `Parent0<u32>`
+   |                    ambiguous `K` from `Parent0<i32>`
+LL |
+LL | fn take0(_: impl Trait0<K = { () }>) {}
+   |                         ^^^^^^^^^^ ambiguous associated constant `K`
+   |
+   = help: consider introducing a new type parameter `T` and adding `where` constraints:
+               where
+                   T: Trait0,
+                   T: Parent0<u32>::K = { () },
+                   T: Parent0<i32>::K = { () }
+
+error[E0222]: ambiguous associated constant `C` in bounds of `Trait1`
+  --> $DIR/assoc-const-eq-ambiguity.rs:16:25
+   |
+LL | trait Parent1 { const C: i32; }
+   |                 ------------ ambiguous `C` from `Parent1`
+LL | trait Parent2 { const C: &'static str; }
+   |                 --------------------- ambiguous `C` from `Parent2`
+LL |
+LL | fn take1(_: impl Trait1<C = "?">) {}
+   |                         ^^^^^^^ ambiguous associated constant `C`
+   |
+   = help: consider introducing a new type parameter `T` and adding `where` constraints:
+               where
+                   T: Trait1,
+                   T: Parent2::C = "?",
+                   T: Parent1::C = "?"
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0222`.
diff --git a/tests/ui/associated-consts/assoc-const-eq-missing.rs b/tests/ui/associated-consts/assoc-const-eq-missing.rs
index 5e029a12df2..f384927e4a3 100644
--- a/tests/ui/associated-consts/assoc-const-eq-missing.rs
+++ b/tests/ui/associated-consts/assoc-const-eq-missing.rs
@@ -11,13 +11,12 @@ impl Foo for Bar {
   const N: usize = 3;
 }
 
-
-fn foo1<F: Foo<Z=3>>() {}
-//~^ ERROR associated type
-fn foo2<F: Foo<Z=usize>>() {}
-//~^ ERROR associated type
-fn foo3<F: Foo<Z=5>>() {}
-//~^ ERROR associated type
+fn foo1<F: Foo<Z = 3>>() {}
+//~^ ERROR associated constant `Z` not found for `Foo`
+fn foo2<F: Foo<Z = usize>>() {}
+//~^ ERROR associated type `Z` not found for `Foo`
+fn foo3<F: Foo<Z = 5>>() {}
+//~^ ERROR associated constant `Z` not found for `Foo`
 
 fn main() {
   foo1::<Bar>();
diff --git a/tests/ui/associated-consts/assoc-const-eq-missing.stderr b/tests/ui/associated-consts/assoc-const-eq-missing.stderr
index b4bd6456c85..318c85dcfd6 100644
--- a/tests/ui/associated-consts/assoc-const-eq-missing.stderr
+++ b/tests/ui/associated-consts/assoc-const-eq-missing.stderr
@@ -1,20 +1,20 @@
-error[E0220]: associated type `Z` not found for `Foo`
-  --> $DIR/assoc-const-eq-missing.rs:15:16
+error[E0220]: associated constant `Z` not found for `Foo`
+  --> $DIR/assoc-const-eq-missing.rs:14:16
    |
-LL | fn foo1<F: Foo<Z=3>>() {}
-   |                ^ associated type `Z` not found
+LL | fn foo1<F: Foo<Z = 3>>() {}
+   |                ^ help: there is an associated constant with a similar name: `N`
 
 error[E0220]: associated type `Z` not found for `Foo`
-  --> $DIR/assoc-const-eq-missing.rs:17:16
+  --> $DIR/assoc-const-eq-missing.rs:16:16
    |
-LL | fn foo2<F: Foo<Z=usize>>() {}
+LL | fn foo2<F: Foo<Z = usize>>() {}
    |                ^ associated type `Z` not found
 
-error[E0220]: associated type `Z` not found for `Foo`
-  --> $DIR/assoc-const-eq-missing.rs:19:16
+error[E0220]: associated constant `Z` not found for `Foo`
+  --> $DIR/assoc-const-eq-missing.rs:18:16
    |
-LL | fn foo3<F: Foo<Z=5>>() {}
-   |                ^ associated type `Z` not found
+LL | fn foo3<F: Foo<Z = 5>>() {}
+   |                ^ help: there is an associated constant with a similar name: `N`
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/associated-consts/assoc-const-eq-ty-alias-noninteracting.rs b/tests/ui/associated-consts/assoc-const-eq-ty-alias-noninteracting.rs
new file mode 100644
index 00000000000..de9008bfcf9
--- /dev/null
+++ b/tests/ui/associated-consts/assoc-const-eq-ty-alias-noninteracting.rs
@@ -0,0 +1,21 @@
+// Regression test for issue #112560.
+// Respect the fact that (associated) types and constants live in different namespaces and
+// therefore equality bounds involving identically named associated items don't conflict if
+// their kind (type vs. const) differs.
+
+// FIXME(fmease): Extend this test to cover supertraits again
+// once #118040 is fixed. See initial version of PR #118360.
+
+// check-pass
+
+#![feature(associated_const_equality)]
+
+trait Trait {
+    type N;
+
+    const N: usize;
+}
+
+fn take(_: impl Trait<N = 0, N = ()>) {}
+
+fn main() {}
diff --git a/tests/ui/associated-consts/assoc-const-ty-mismatch.rs b/tests/ui/associated-consts/assoc-const-ty-mismatch.rs
index c5d78469e95..7211637659b 100644
--- a/tests/ui/associated-consts/assoc-const-ty-mismatch.rs
+++ b/tests/ui/associated-consts/assoc-const-ty-mismatch.rs
@@ -2,30 +2,30 @@
 #![allow(unused)]
 
 pub trait Foo {
-  const N: usize;
+    const N: usize;
 }
 
 pub trait FooTy {
-  type T;
+    type T;
 }
 
 pub struct Bar;
 
 impl Foo for Bar {
-  const N: usize = 3;
+    const N: usize = 3;
 }
 
 impl FooTy for Bar {
-  type T = usize;
+    type T = usize;
 }
 
 
-fn foo<F: Foo<N=usize>>() {}
-//~^ ERROR expected associated constant bound, found type
-fn foo2<F: FooTy<T=3usize>>() {}
-//~^ ERROR expected associated type bound, found constant
+fn foo<F: Foo<N = usize>>() {}
+//~^ ERROR expected constant, found type
+fn foo2<F: FooTy<T = 3usize>>() {}
+//~^ ERROR expected type, found constant
 
 fn main() {
-  foo::<Bar>();
-  foo2::<Bar>();
+    foo::<Bar>();
+    foo2::<Bar>();
 }
diff --git a/tests/ui/associated-consts/assoc-const-ty-mismatch.stderr b/tests/ui/associated-consts/assoc-const-ty-mismatch.stderr
index 11198729e38..b844cfc4ae4 100644
--- a/tests/ui/associated-consts/assoc-const-ty-mismatch.stderr
+++ b/tests/ui/associated-consts/assoc-const-ty-mismatch.stderr
@@ -1,26 +1,30 @@
-error: expected associated constant bound, found type
-  --> $DIR/assoc-const-ty-mismatch.rs:23:15
+error: expected constant, found type
+  --> $DIR/assoc-const-ty-mismatch.rs:23:19
    |
-LL | fn foo<F: Foo<N=usize>>() {}
-   |               ^^^^^^^
+LL | fn foo<F: Foo<N = usize>>() {}
+   |               -   ^^^^^ unexpected type
+   |               |
+   |               expected a constant because of this associated constant
    |
-note: associated constant defined here
-  --> $DIR/assoc-const-ty-mismatch.rs:5:3
+note: the associated constant is defined here
+  --> $DIR/assoc-const-ty-mismatch.rs:5:5
    |
-LL |   const N: usize;
-   |   ^^^^^^^^^^^^^^
+LL |     const N: usize;
+   |     ^^^^^^^^^^^^^^
 
-error: expected associated type bound, found constant
-  --> $DIR/assoc-const-ty-mismatch.rs:25:18
+error: expected type, found constant
+  --> $DIR/assoc-const-ty-mismatch.rs:25:22
    |
-LL | fn foo2<F: FooTy<T=3usize>>() {}
-   |                  ^^^^^^^^
+LL | fn foo2<F: FooTy<T = 3usize>>() {}
+   |                  -   ^^^^^^ unexpected constant
+   |                  |
+   |                  expected a type because of this associated type
    |
-note: associated type defined here
-  --> $DIR/assoc-const-ty-mismatch.rs:9:3
+note: the associated type is defined here
+  --> $DIR/assoc-const-ty-mismatch.rs:9:5
    |
-LL |   type T;
-   |   ^^^^^^
+LL |     type T;
+   |     ^^^^^^
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/associated-consts/shadowed-const.rs b/tests/ui/associated-consts/shadowed-const.rs
index cfdb391d39d..d9b565742d4 100644
--- a/tests/ui/associated-consts/shadowed-const.rs
+++ b/tests/ui/associated-consts/shadowed-const.rs
@@ -17,7 +17,7 @@ trait Baz2: Foo {
 trait Baz3 {
   const BAR: usize;
   const QUX: Self::BAR;
-  //~^ ERROR found associated const
+  //~^ ERROR expected type, found constant
 }
 
 fn main() {}
diff --git a/tests/ui/associated-consts/shadowed-const.stderr b/tests/ui/associated-consts/shadowed-const.stderr
index a01a9ae561f..2db645b3c96 100644
--- a/tests/ui/associated-consts/shadowed-const.stderr
+++ b/tests/ui/associated-consts/shadowed-const.stderr
@@ -1,8 +1,14 @@
-error: found associated const `BAR` when type was expected
-  --> $DIR/shadowed-const.rs:19:14
+error: expected type, found constant
+  --> $DIR/shadowed-const.rs:19:20
    |
 LL |   const QUX: Self::BAR;
-   |              ^^^^^^^^^
+   |                    ^^^ unexpected constant
+   |
+note: the associated constant is defined here
+  --> $DIR/shadowed-const.rs:18:3
+   |
+LL |   const BAR: usize;
+   |   ^^^^^^^^^^^^^^^^
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/associated-type-bounds/consts.rs b/tests/ui/associated-type-bounds/consts.rs
index 9b95b1b52c0..8f90c36ed45 100644
--- a/tests/ui/associated-type-bounds/consts.rs
+++ b/tests/ui/associated-type-bounds/consts.rs
@@ -1,7 +1,7 @@
 #![feature(associated_type_bounds)]
 
 pub fn accept(_: impl Trait<K: Copy>) {}
-//~^ ERROR expected associated type, found associated constant
+//~^ ERROR expected type, found constant
 
 pub trait Trait {
     const K: i32;
diff --git a/tests/ui/associated-type-bounds/consts.stderr b/tests/ui/associated-type-bounds/consts.stderr
index eef24c8827b..7f9fe5e500a 100644
--- a/tests/ui/associated-type-bounds/consts.stderr
+++ b/tests/ui/associated-type-bounds/consts.stderr
@@ -1,10 +1,16 @@
-error: expected associated type, found associated constant
+error: expected type, found constant
   --> $DIR/consts.rs:3:29
    |
 LL | pub fn accept(_: impl Trait<K: Copy>) {}
-   |                             ^
+   |                             ^------ bounds are not allowed on associated constants
+   |                             |
+   |                             unexpected constant
    |
-   = note: trait bounds not allowed on associated constant
+note: the associated constant is defined here
+  --> $DIR/consts.rs:7:5
+   |
+LL |     const K: i32;
+   |     ^^^^^^^^^^^^
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/associated-type-bounds/issue-99828.rs b/tests/ui/associated-type-bounds/issue-99828.rs
index 7b711283f5b..67ba50f3cbc 100644
--- a/tests/ui/associated-type-bounds/issue-99828.rs
+++ b/tests/ui/associated-type-bounds/issue-99828.rs
@@ -1,5 +1,5 @@
 fn get_iter(vec: &[i32]) -> impl Iterator<Item = {}> + '_ {
-    //~^ ERROR expected associated type bound, found constant
+    //~^ ERROR expected type, found constant
     //~| ERROR associated const equality is incomplete
     vec.iter()
 }
diff --git a/tests/ui/associated-type-bounds/issue-99828.stderr b/tests/ui/associated-type-bounds/issue-99828.stderr
index dc93c47dace..8813baf84de 100644
--- a/tests/ui/associated-type-bounds/issue-99828.stderr
+++ b/tests/ui/associated-type-bounds/issue-99828.stderr
@@ -7,13 +7,15 @@ LL | fn get_iter(vec: &[i32]) -> impl Iterator<Item = {}> + '_ {
    = note: see issue #92827 <https://github.com/rust-lang/rust/issues/92827> for more information
    = help: add `#![feature(associated_const_equality)]` to the crate attributes to enable
 
-error: expected associated type bound, found constant
-  --> $DIR/issue-99828.rs:1:43
+error: expected type, found constant
+  --> $DIR/issue-99828.rs:1:50
    |
 LL | fn get_iter(vec: &[i32]) -> impl Iterator<Item = {}> + '_ {
-   |                                           ^^^^^^^^^
+   |                                           ----   ^^ unexpected constant
+   |                                           |
+   |                                           expected a type because of this associated type
    |
-note: associated type defined here
+note: the associated type is defined here
   --> $SRC_DIR/core/src/iter/traits/iterator.rs:LL:COL
 
 error: aborting due to 2 previous errors
diff --git a/tests/ui/associated-type-bounds/return-type-notation/missing.rs b/tests/ui/associated-type-bounds/return-type-notation/missing.rs
index 0679b96f6c5..e6270ec3166 100644
--- a/tests/ui/associated-type-bounds/return-type-notation/missing.rs
+++ b/tests/ui/associated-type-bounds/return-type-notation/missing.rs
@@ -8,6 +8,6 @@ trait Trait {
 }
 
 fn bar<T: Trait<methid(): Send>>() {}
-//~^ ERROR cannot find associated function `methid` for `Trait`
+//~^ ERROR associated function `methid` not found for `Trait`
 
 fn main() {}
diff --git a/tests/ui/associated-type-bounds/return-type-notation/missing.stderr b/tests/ui/associated-type-bounds/return-type-notation/missing.stderr
index 3ca5e66866d..db9cb9f49a3 100644
--- a/tests/ui/associated-type-bounds/return-type-notation/missing.stderr
+++ b/tests/ui/associated-type-bounds/return-type-notation/missing.stderr
@@ -7,11 +7,12 @@ LL | #![feature(return_type_notation)]
    = note: see issue #109417 <https://github.com/rust-lang/rust/issues/109417> for more information
    = note: `#[warn(incomplete_features)]` on by default
 
-error: cannot find associated function `methid` for `Trait`
+error[E0220]: associated function `methid` not found for `Trait`
   --> $DIR/missing.rs:10:17
    |
 LL | fn bar<T: Trait<methid(): Send>>() {}
-   |                 ^^^^^^^^^^^^^^
+   |                 ^^^^^^ help: there is an associated function with a similar name: `method`
 
 error: aborting due to 1 previous error; 1 warning emitted
 
+For more information about this error, try `rustc --explain E0220`.
diff --git a/tests/ui/async-await/return-type-notation/super-method-bound-ambig.rs b/tests/ui/async-await/return-type-notation/super-method-bound-ambig.rs
index 891b30638ee..73c08531599 100644
--- a/tests/ui/async-await/return-type-notation/super-method-bound-ambig.rs
+++ b/tests/ui/async-await/return-type-notation/super-method-bound-ambig.rs
@@ -23,7 +23,7 @@ impl Foo for () {}
 fn test<T>()
 where
     T: Foo<test(): Send>,
-    //~^ ERROR ambiguous associated function `test` for `Foo`
+    //~^ ERROR ambiguous associated function `test` in bounds of `Foo`
 {
 }
 
diff --git a/tests/ui/async-await/return-type-notation/super-method-bound-ambig.stderr b/tests/ui/async-await/return-type-notation/super-method-bound-ambig.stderr
index 7eaf3b82d98..4003aad6d03 100644
--- a/tests/ui/async-await/return-type-notation/super-method-bound-ambig.stderr
+++ b/tests/ui/async-await/return-type-notation/super-method-bound-ambig.stderr
@@ -7,13 +7,18 @@ LL | #![feature(return_type_notation)]
    = note: see issue #109417 <https://github.com/rust-lang/rust/issues/109417> for more information
    = note: `#[warn(incomplete_features)]` on by default
 
-error: ambiguous associated function `test` for `Foo`
+error[E0221]: ambiguous associated function `test` in bounds of `Foo`
   --> $DIR/super-method-bound-ambig.rs:25:12
    |
+LL |     async fn test();
+   |     ---------------- ambiguous `test` from `for<'a> Super1<'a>`
+...
+LL |     async fn test();
+   |     ---------------- ambiguous `test` from `Super2`
+...
 LL |     T: Foo<test(): Send>,
-   |            ^^^^^^^^^^^^
-   |
-   = note: `test` is declared in two supertraits: `Super2` and `Super1<'a>`
+   |            ^^^^^^^^^^^^ ambiguous associated function `test`
 
 error: aborting due to 1 previous error; 1 warning emitted
 
+For more information about this error, try `rustc --explain E0221`.
diff --git a/tests/ui/const-generics/assoc_const_eq_diagnostic.rs b/tests/ui/const-generics/assoc_const_eq_diagnostic.rs
index bf8202ac152..d51696f9ebd 100644
--- a/tests/ui/const-generics/assoc_const_eq_diagnostic.rs
+++ b/tests/ui/const-generics/assoc_const_eq_diagnostic.rs
@@ -9,9 +9,9 @@ pub trait Parse {
 }
 
 pub trait CoolStuff: Parse<MODE = Mode::Cool> {}
-//~^ ERROR expected associated constant bound
-//~| ERROR expected associated constant bound
-//~| ERROR expected type
+//~^ ERROR expected type, found variant
+//~| ERROR expected constant, found type
+//~| ERROR expected constant, found type
 
 fn no_help() -> Mode::Cool {}
 //~^ ERROR expected type, found variant
diff --git a/tests/ui/const-generics/assoc_const_eq_diagnostic.stderr b/tests/ui/const-generics/assoc_const_eq_diagnostic.stderr
index 6d5cd45479e..3d724bb1642 100644
--- a/tests/ui/const-generics/assoc_const_eq_diagnostic.stderr
+++ b/tests/ui/const-generics/assoc_const_eq_diagnostic.stderr
@@ -16,30 +16,42 @@ LL | fn no_help() -> Mode::Cool {}
    |                 not a type
    |                 help: try using the variant's enum: `Mode`
 
-error: expected associated constant bound, found type
-  --> $DIR/assoc_const_eq_diagnostic.rs:11:28
+error: expected constant, found type
+  --> $DIR/assoc_const_eq_diagnostic.rs:11:35
    |
 LL | pub trait CoolStuff: Parse<MODE = Mode::Cool> {}
-   |                            ^^^^^^^^^^^^^^^^^ help: if equating a const, try wrapping with braces: `MODE = { const }`
+   |                            ----   ^^^^^^^^^^ unexpected type
+   |                            |
+   |                            expected a constant because of this associated constant
    |
-note: associated constant defined here
+note: the associated constant is defined here
   --> $DIR/assoc_const_eq_diagnostic.rs:8:5
    |
 LL |     const MODE: Mode;
    |     ^^^^^^^^^^^^^^^^
+help: consider adding braces here
+   |
+LL | pub trait CoolStuff: Parse<MODE = { Mode::Cool }> {}
+   |                                   +            +
 
-error: expected associated constant bound, found type
-  --> $DIR/assoc_const_eq_diagnostic.rs:11:28
+error: expected constant, found type
+  --> $DIR/assoc_const_eq_diagnostic.rs:11:35
    |
 LL | pub trait CoolStuff: Parse<MODE = Mode::Cool> {}
-   |                            ^^^^^^^^^^^^^^^^^ help: if equating a const, try wrapping with braces: `MODE = { const }`
+   |                            ----   ^^^^^^^^^^ unexpected type
+   |                            |
+   |                            expected a constant because of this associated constant
    |
-note: associated constant defined here
+note: the associated constant is defined here
   --> $DIR/assoc_const_eq_diagnostic.rs:8:5
    |
 LL |     const MODE: Mode;
    |     ^^^^^^^^^^^^^^^^
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: consider adding braces here
+   |
+LL | pub trait CoolStuff: Parse<MODE = { Mode::Cool }> {}
+   |                                   +            +
 
 error: aborting due to 4 previous errors
 
diff --git a/tests/ui/error-codes/E0221.stderr b/tests/ui/error-codes/E0221.stderr
index e600acf7834..07e7485b67e 100644
--- a/tests/ui/error-codes/E0221.stderr
+++ b/tests/ui/error-codes/E0221.stderr
@@ -28,7 +28,7 @@ LL |     fn test() {
 LL |         let _: Self::Err;
    |                ^^^^^^^^^ ambiguous associated type `Err`
    |
-   = note: associated type `Self` could derive from `FromStr`
+   = note: associated type `Err` could derive from `FromStr`
 help: use fully-qualified syntax to disambiguate
    |
 LL |         let _: <Self as My>::Err;
diff --git a/tests/ui/feature-gates/feature-gate-return_type_notation.cfg.stderr b/tests/ui/feature-gates/feature-gate-return_type_notation.cfg.stderr
index 1bdb2574ead..a15b01618f5 100644
--- a/tests/ui/feature-gates/feature-gate-return_type_notation.cfg.stderr
+++ b/tests/ui/feature-gates/feature-gate-return_type_notation.cfg.stderr
@@ -15,13 +15,18 @@ LL | fn foo<T: Trait<m(): Send>>() {}
    |                  |
    |                  help: remove these parentheses
 
-error[E0220]: associated type `m` not found for `Trait`
+error: expected type, found function
   --> $DIR/feature-gate-return_type_notation.rs:14:17
    |
 LL | fn foo<T: Trait<m(): Send>>() {}
-   |                 ^ associated type `m` not found
+   |                 ^ unexpected function
+   |
+note: the associated function is defined here
+  --> $DIR/feature-gate-return_type_notation.rs:10:5
+   |
+LL |     async fn m();
+   |     ^^^^^^^^^^^^^
 
 error: aborting due to 3 previous errors
 
-Some errors have detailed explanations: E0220, E0658.
-For more information about an error, try `rustc --explain E0220`.
+For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/feature-gates/feature-gate-return_type_notation.rs b/tests/ui/feature-gates/feature-gate-return_type_notation.rs
index 86e2c48e188..60ac9f8d4f1 100644
--- a/tests/ui/feature-gates/feature-gate-return_type_notation.rs
+++ b/tests/ui/feature-gates/feature-gate-return_type_notation.rs
@@ -14,7 +14,7 @@ trait Trait {
 fn foo<T: Trait<m(): Send>>() {}
 //[cfg]~^ ERROR return type notation is experimental
 //[cfg]~| ERROR parenthesized generic arguments cannot be used in associated type constraints
-//[cfg]~| ERROR associated type `m` not found for `Trait`
+//[cfg]~| ERROR expected type, found function
 //[no]~^^^^ WARN return type notation is experimental
 //[no]~| WARN unstable syntax can change at any point in the future, causing a hard error!
 
-- 
cgit 1.4.1-3-g733a5


From 0158404e78f26e75bb234782d7bc58a9eda9b00c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 8 Dec 2023 08:40:12 +1100
Subject: Remove `BorrowckAnalyses`.

This results in two non-generic types being used: `BorrowckResults` and
`BorrowckFlowState`. It's a net reduction in lines of code, and a little
easier to read.
---
 compiler/rustc_borrowck/src/dataflow.rs | 46 +++++++++++++--------------------
 1 file changed, 18 insertions(+), 28 deletions(-)

diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index 5a42d9325fa..bc80a88e1e6 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -11,43 +11,33 @@ use rustc_middle::ty::TyCtxt;
 use rustc_mir_dataflow::impls::{EverInitializedPlaces, MaybeUninitializedPlaces};
 use rustc_mir_dataflow::ResultsVisitable;
 use rustc_mir_dataflow::{self, fmt::DebugWithContext, GenKill};
-use rustc_mir_dataflow::{Analysis, Direction, Results};
+use rustc_mir_dataflow::{Analysis, AnalysisDomain, Results};
 use std::fmt;
 
 use crate::{places_conflict, BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext};
 
-/// A tuple with named fields that can hold either the results or the transient state of the
-/// dataflow analyses used by the borrow checker.
-#[derive(Debug)]
-pub struct BorrowckAnalyses<B, U, E> {
-    pub borrows: B,
-    pub uninits: U,
-    pub ever_inits: E,
-}
-
 /// The results of the dataflow analyses used by the borrow checker.
-pub type BorrowckResults<'mir, 'tcx> = BorrowckAnalyses<
-    Results<'tcx, Borrows<'mir, 'tcx>>,
-    Results<'tcx, MaybeUninitializedPlaces<'mir, 'tcx>>,
-    Results<'tcx, EverInitializedPlaces<'mir, 'tcx>>,
->;
+pub struct BorrowckResults<'mir, 'tcx> {
+    pub(crate) borrows: Results<'tcx, Borrows<'mir, 'tcx>>,
+    pub(crate) uninits: Results<'tcx, MaybeUninitializedPlaces<'mir, 'tcx>>,
+    pub(crate) ever_inits: Results<'tcx, EverInitializedPlaces<'mir, 'tcx>>,
+}
 
 /// The transient state of the dataflow analyses used by the borrow checker.
-pub type BorrowckFlowState<'mir, 'tcx> =
-    <BorrowckResults<'mir, 'tcx> as ResultsVisitable<'tcx>>::FlowState;
-
-impl<'tcx, B, U, E, D: Direction> ResultsVisitable<'tcx>
-    for BorrowckAnalyses<Results<'tcx, B>, Results<'tcx, U>, Results<'tcx, E>>
-where
-    B: Analysis<'tcx, Direction = D>,
-    U: Analysis<'tcx, Direction = D>,
-    E: Analysis<'tcx, Direction = D>,
-{
-    type Direction = D;
-    type FlowState = BorrowckAnalyses<B::Domain, U::Domain, E::Domain>;
+#[derive(Debug)]
+pub struct BorrowckFlowState<'mir, 'tcx> {
+    pub(crate) borrows: <Borrows<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) uninits: <MaybeUninitializedPlaces<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) ever_inits: <EverInitializedPlaces<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+}
+
+impl<'mir, 'tcx> ResultsVisitable<'tcx> for BorrowckResults<'mir, 'tcx> {
+    // All three analyses are forward, but we have to use just one here.
+    type Direction = <Borrows<'mir, 'tcx> as AnalysisDomain<'tcx>>::Direction;
+    type FlowState = BorrowckFlowState<'mir, 'tcx>;
 
     fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
-        BorrowckAnalyses {
+        BorrowckFlowState {
             borrows: self.borrows.analysis.bottom_value(body),
             uninits: self.uninits.analysis.bottom_value(body),
             ever_inits: self.ever_inits.analysis.bottom_value(body),
-- 
cgit 1.4.1-3-g733a5


From 4b364b6f0f1d7d6db1b289777068a11c1021bc24 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 8 Dec 2023 09:35:57 +1100
Subject: Tweak `GenKillAnalysis`.

`GenKillAnalysis` has five methods that take a transfer function arg:
- `statement_effect`
- `before_statement_effect`
- `terminator_effect`
- `before_terminator_effect`
- `call_return_effect`

All the transfer function args have type `&mut impl GenKill<Self::Idx>`,
except for `terminator_effect`, which takes the simpler `Self::Domain`.

But only the first two need to be `impl GenKill`. The other
three can all be `Self::Domain`, just like `Analysis`. So this commit
changes the last two to take `Self::Domain`, making `GenKillAnalysis`
and `Analysis` more similar.

(Another idea would be to make all these methods `impl GenKill`. But
that doesn't work: `MaybeInitializedPlaces::terminator_effect` requires
the arg be `Self::Domain` so that `self_is_unwind_dead(place, state)`
can be called on it.)
---
 compiler/rustc_borrowck/src/dataflow.rs             |  4 ++--
 compiler/rustc_mir_dataflow/src/framework/mod.rs    | 21 ++++++++++++---------
 .../rustc_mir_dataflow/src/impls/borrowed_locals.rs |  2 +-
 .../rustc_mir_dataflow/src/impls/initialized.rs     |  8 ++++----
 compiler/rustc_mir_dataflow/src/impls/liveness.rs   |  2 +-
 .../src/impls/storage_liveness.rs                   |  8 ++++----
 6 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index bc80a88e1e6..1b544b53012 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -585,7 +585,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
 
     fn before_terminator_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _terminator: &mir::Terminator<'tcx>,
         location: Location,
     ) {
@@ -612,7 +612,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_mir_dataflow/src/framework/mod.rs b/compiler/rustc_mir_dataflow/src/framework/mod.rs
index b7dfbe0710d..09cdb055a3e 100644
--- a/compiler/rustc_mir_dataflow/src/framework/mod.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/mod.rs
@@ -248,18 +248,19 @@ pub trait Analysis<'tcx>: AnalysisDomain<'tcx> {
 
 /// A gen/kill dataflow problem.
 ///
-/// Each method in this trait has a corresponding one in `Analysis`. However, these methods only
-/// allow modification of the dataflow state via "gen" and "kill" operations. By defining transfer
-/// functions for each statement in this way, the transfer function for an entire basic block can
-/// be computed efficiently.
+/// Each method in this trait has a corresponding one in `Analysis`. However, the first two methods
+/// here only allow modification of the dataflow state via "gen" and "kill" operations. By defining
+/// transfer functions for each statement in this way, the transfer function for an entire basic
+/// block can be computed efficiently. The remaining methods match up with `Analysis` exactly.
 ///
-/// `Analysis` is automatically implemented for all implementers of `GenKillAnalysis`.
+/// `Analysis` is automatically implemented for all implementers of `GenKillAnalysis` via a blanket
+/// impl below.
 pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     type Idx: Idx;
 
     fn domain_size(&self, body: &mir::Body<'tcx>) -> usize;
 
-    /// See `Analysis::apply_statement_effect`.
+    /// See `Analysis::apply_statement_effect`. Note how the second arg differs.
     fn statement_effect(
         &mut self,
         trans: &mut impl GenKill<Self::Idx>,
@@ -267,7 +268,8 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
         location: Location,
     );
 
-    /// See `Analysis::apply_before_statement_effect`.
+    /// See `Analysis::apply_before_statement_effect`. Note how the second arg
+    /// differs.
     fn before_statement_effect(
         &mut self,
         _trans: &mut impl GenKill<Self::Idx>,
@@ -287,7 +289,7 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     /// See `Analysis::apply_before_terminator_effect`.
     fn before_terminator_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _terminator: &mir::Terminator<'tcx>,
         _location: Location,
     ) {
@@ -298,7 +300,7 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     /// See `Analysis::apply_call_return_effect`.
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         block: BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     );
@@ -313,6 +315,7 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     }
 }
 
+// Blanket impl: any impl of `GenKillAnalysis` automatically impls `Analysis`.
 impl<'tcx, A> Analysis<'tcx> for A
 where
     A: GenKillAnalysis<'tcx>,
diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
index 01acc380fa3..693994b5da7 100644
--- a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
@@ -62,7 +62,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeBorrowedLocals {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_mir_dataflow/src/impls/initialized.rs b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
index b050e963d8e..6653b99b3f5 100644
--- a/compiler/rustc_mir_dataflow/src/impls/initialized.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
@@ -376,7 +376,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -499,7 +499,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -617,7 +617,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for DefinitelyInitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -712,7 +712,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for EverInitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         block: mir::BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
index c3fdca1641a..04bae6ae2fe 100644
--- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
@@ -69,7 +69,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeLiveLocals {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
index 26fc903973f..646c70eb88f 100644
--- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
@@ -72,7 +72,7 @@ impl<'tcx, 'a> crate::GenKillAnalysis<'tcx> for MaybeStorageLive<'a> {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -144,7 +144,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeStorageDead {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -238,7 +238,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> {
 
     fn before_terminator_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         terminator: &Terminator<'tcx>,
         loc: Location,
     ) {
@@ -334,7 +334,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
-- 
cgit 1.4.1-3-g733a5


From a1b9a599584e9044da30bfd8fbc0d16ca72c009e Mon Sep 17 00:00:00 2001
From: Tyler Mandry <tmandry@google.com>
Date: Tue, 7 Nov 2023 18:11:07 -0800
Subject: Build Fuchsia in CI

---
 .github/workflows/ci.yml                           |  3 +
 src/bootstrap/src/core/build_steps/compile.rs      |  2 +-
 .../docker/host-x86_64/dist-various-2/Dockerfile   |  4 +-
 .../dist-various-2/build-fuchsia-toolchain.sh      | 60 -------------------
 src/ci/docker/host-x86_64/dist-various-2/shared.sh | 35 -----------
 .../host-x86_64/x86_64-gnu-integration/Dockerfile  | 68 ++++++++++++++++++++++
 .../x86_64-gnu-integration/build-fuchsia.sh        | 51 ++++++++++++++++
 src/ci/docker/scripts/build-fuchsia-toolchain.sh   | 60 +++++++++++++++++++
 src/ci/docker/scripts/shared.sh                    | 42 +++++++++++++
 src/ci/github-actions/ci.yml                       |  4 ++
 src/ci/shared.sh                                   |  2 +-
 11 files changed, 232 insertions(+), 99 deletions(-)
 delete mode 100755 src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh
 delete mode 100644 src/ci/docker/host-x86_64/dist-various-2/shared.sh
 create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile
 create mode 100755 src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh
 create mode 100755 src/ci/docker/scripts/build-fuchsia-toolchain.sh
 create mode 100644 src/ci/docker/scripts/shared.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a0d9eb8c452..4cf603519b1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -288,6 +288,9 @@ jobs:
           - name: x86_64-gnu-aux
             os: ubuntu-20.04-4core-16gb
             env: {}
+          - name: x86_64-gnu-integration
+            os: ubuntu-20.04-16core-64gb
+            env: {}
           - name: x86_64-gnu-debug
             os: ubuntu-20.04-8core-32gb
             env: {}
diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index c253dd2c6aa..1ab488ef656 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -1722,7 +1722,7 @@ impl Step for Assemble {
             let dst_exe = exe("rust-lld", target_compiler.host);
             builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe));
             let self_contained_lld_dir = libdir_bin.join("gcc-ld");
-            t!(fs::create_dir(&self_contained_lld_dir));
+            t!(fs::create_dir_all(&self_contained_lld_dir));
             let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper {
                 compiler: build_compiler,
                 target: target_compiler.host,
diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile
index 3f4e025111a..5f1fec74bed 100644
--- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile
@@ -76,8 +76,8 @@ RUN env \
     rm -rf /build/*
 
 WORKDIR /tmp
-COPY host-x86_64/dist-various-2/shared.sh /tmp/
-COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/
+COPY scripts/shared.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
 COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/
 RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386  pc
diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh b/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh
deleted file mode 100755
index d762b4672c6..00000000000
--- a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-source shared.sh
-
-FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64
-FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC
-FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179
-FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk
-CLANG_DOWNLOAD_URL=\
-https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64
-CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC
-CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b
-
-install_clang() {
-  mkdir -p clang_download
-  pushd clang_download > /dev/null
-
-  # Download clang+llvm
-  curl -LO "${CLANG_DOWNLOAD_URL}/+/${CLANG_DOWNLOAD_ID}"
-  echo "$(echo ${CLANG_DOWNLOAD_SHA256}) ${CLANG_DOWNLOAD_ID}" | sha256sum --check --status
-  unzip -qq ${CLANG_DOWNLOAD_ID} -d clang-linux-amd64
-
-  # Other dists currently depend on our Clang... moving into /usr/local for other
-  #  dist usage instead of a Fuchsia /usr/local directory
-  chmod -R 777 clang-linux-amd64/.
-  cp -a clang-linux-amd64/. /usr/local
-
-  # CFLAGS and CXXFLAGS env variables in main Dockerfile handle sysroot linking
-  for arch in x86_64 aarch64; do
-    for tool in clang clang++; do
-      ln -s /usr/local/bin/${tool} /usr/local/bin/${arch}-unknown-fuchsia-${tool}
-    done
-    ln -s /usr/local/bin/llvm-ar /usr/local/bin/${arch}-unknown-fuchsia-ar
-  done
-
-  popd > /dev/null
-  rm -rf clang_download
-}
-
-install_zircon_libs() {
-  mkdir -p zircon
-  pushd zircon > /dev/null
-
-  # Download Fuchsia SDK (with Zircon libs)
-  curl -LO "${FUCHSIA_SDK_URL}/+/${FUCHSIA_SDK_ID}"
-  echo "$(echo ${FUCHSIA_SDK_SHA256}) ${FUCHSIA_SDK_ID}" | sha256sum --check --status
-  unzip -qq ${FUCHSIA_SDK_ID} -d core-linux-amd64
-
-  # Moving SDK into Docker's user-space
-  mkdir -p ${FUCHSIA_SDK_USR_DIR}
-  chmod -R 777 core-linux-amd64/.
-  cp -r core-linux-amd64/* ${FUCHSIA_SDK_USR_DIR}
-
-  popd > /dev/null
-  rm -rf zircon
-}
-
-hide_output install_clang
-hide_output install_zircon_libs
diff --git a/src/ci/docker/host-x86_64/dist-various-2/shared.sh b/src/ci/docker/host-x86_64/dist-various-2/shared.sh
deleted file mode 100644
index 291f26bdaeb..00000000000
--- a/src/ci/docker/host-x86_64/dist-various-2/shared.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-hide_output() {
-  { set +x; } 2>/dev/null
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  "$@" &> /tmp/build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  set -x
-}
-
-# Copied from ../../shared.sh
-function retry {
-  echo "Attempting with retry:" "$@"
-  local n=1
-  local max=5
-  while true; do
-    "$@" && break || {
-      if [[ $n -lt $max ]]; then
-        sleep $n  # don't retry immediately
-        ((n++))
-        echo "Command failed. Attempt $n/$max:"
-      else
-        echo "The command has failed after $n attempts."
-        return 1
-      fi
-    }
-  done
-}
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile
new file mode 100644
index 00000000000..ba65ba9bed4
--- /dev/null
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile
@@ -0,0 +1,68 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  build-essential \
+  g++ \
+  make \
+  ninja-build \
+  file \
+  curl \
+  ca-certificates \
+  python3 \
+  git \
+  cmake \
+  libssl-dev \
+  sudo \
+  xz-utils \
+  pkg-config \
+  unzip \
+  && rm -rf /var/lib/apt/lists/*
+
+# Duplicated in dist-various-2 Dockerfile.
+# FIXME: Move to canonical triple
+ENV \
+    AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \
+    CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \
+    CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \
+    CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib"
+
+WORKDIR /tmp
+COPY scripts/shared.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
+RUN /tmp/build-fuchsia-toolchain.sh
+
+ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar
+ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \
+  -C panic=abort \
+  -C force-unwind-tables=yes \
+  -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib
+
+ENV TARGETS=x86_64-fuchsia
+ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_INSTALL_DIR /checkout/obj/install
+RUN mkdir -p $RUST_INSTALL_DIR/etc
+
+ENV RUST_CONFIGURE_ARGS \
+  --prefix=$RUST_INSTALL_DIR \
+  --sysconfdir=etc \
+  --enable-lld \
+  --llvm-libunwind=in-tree \
+  --enable-extended \
+  --disable-docs \
+  --set target.x86_64-fuchsia.cc=/usr/local/bin/clang \
+  --set target.x86_64-fuchsia.cxx=/usr/local/bin/clang++ \
+  --set target.x86_64-fuchsia.ar=/usr/local/bin/llvm-ar \
+  --set target.x86_64-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
+  --set target.x86_64-fuchsia.linker=/usr/local/bin/ld.lld
+ENV SCRIPT \
+    python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
+    bash ../src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh
new file mode 100755
index 00000000000..4a246f591d7
--- /dev/null
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Downloads and builds the Fuchsia operating system using a toolchain installed
+# in $RUST_INSTALL_DIR.
+
+set -euf -o pipefail
+
+INTEGRATION_SHA=66793c4894bf6204579bbee3b79956335f31c768
+PICK_REFS=()
+
+checkout=fuchsia
+jiri=.jiri_root/bin/jiri
+
+set -x
+
+# This script will:
+# - create a directory named "fuchsia" if it does not exist
+# - download "jiri" to "fuchsia/.jiri_root/bin"
+curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \
+    | base64 --decode \
+    | bash -s $checkout
+
+cd $checkout
+
+$jiri init \
+    -partial=true \
+    -analytics-opt=false \
+    .
+
+$jiri import \
+    -name=integration \
+    -revision=$INTEGRATION_SHA \
+    -overwrite=true \
+    flower \
+    "https://fuchsia.googlesource.com/integration"
+
+if [ -d ".git" ]; then
+    # Wipe out any local changes if we're reusing a checkout.
+    git checkout --force JIRI_HEAD
+fi
+
+$jiri update -autoupdate=false
+
+echo integration commit = $(git -C integration rev-parse HEAD)
+
+for git_ref in "${PICK_REFS[@]}"; do
+    git fetch https://fuchsia.googlesource.com/fuchsia $git_ref
+    git cherry-pick --no-commit FETCH_HEAD
+done
+
+bash scripts/rust/build_fuchsia_from_rust_ci.sh
diff --git a/src/ci/docker/scripts/build-fuchsia-toolchain.sh b/src/ci/docker/scripts/build-fuchsia-toolchain.sh
new file mode 100755
index 00000000000..beea2f522fd
--- /dev/null
+++ b/src/ci/docker/scripts/build-fuchsia-toolchain.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -ex
+source shared.sh
+
+FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64
+FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC
+FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7
+FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk
+CLANG_DOWNLOAD_URL=\
+https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64
+CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C
+CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd
+
+install_clang() {
+  mkdir -p clang_download
+  pushd clang_download > /dev/null
+
+  # Download clang+llvm
+  curl -LO "${CLANG_DOWNLOAD_URL}/+/${CLANG_DOWNLOAD_ID}"
+  echo "$(echo ${CLANG_DOWNLOAD_SHA256}) ${CLANG_DOWNLOAD_ID}" | sha256sum --check --status
+  unzip -qq ${CLANG_DOWNLOAD_ID} -d clang-linux-amd64
+
+  # Other dists currently depend on our Clang... moving into /usr/local for other
+  #  dist usage instead of a Fuchsia /usr/local directory
+  chmod -R 777 clang-linux-amd64/.
+  cp -a clang-linux-amd64/. /usr/local
+
+  # CFLAGS and CXXFLAGS env variables in main Dockerfile handle sysroot linking
+  for arch in x86_64 aarch64; do
+    for tool in clang clang++; do
+      ln -s /usr/local/bin/${tool} /usr/local/bin/${arch}-unknown-fuchsia-${tool}
+    done
+    ln -s /usr/local/bin/llvm-ar /usr/local/bin/${arch}-unknown-fuchsia-ar
+  done
+
+  popd > /dev/null
+  rm -rf clang_download
+}
+
+install_zircon_libs() {
+  mkdir -p zircon
+  pushd zircon > /dev/null
+
+  # Download Fuchsia SDK (with Zircon libs)
+  curl -LO "${FUCHSIA_SDK_URL}/+/${FUCHSIA_SDK_ID}"
+  echo "$(echo ${FUCHSIA_SDK_SHA256}) ${FUCHSIA_SDK_ID}" | sha256sum --check --status
+  unzip -qq ${FUCHSIA_SDK_ID} -d core-linux-amd64
+
+  # Moving SDK into Docker's user-space
+  mkdir -p ${FUCHSIA_SDK_USR_DIR}
+  chmod -R 777 core-linux-amd64/.
+  cp -r core-linux-amd64/* ${FUCHSIA_SDK_USR_DIR}
+
+  popd > /dev/null
+  rm -rf zircon
+}
+
+hide_output install_clang
+hide_output install_zircon_libs
diff --git a/src/ci/docker/scripts/shared.sh b/src/ci/docker/scripts/shared.sh
new file mode 100644
index 00000000000..9969659088d
--- /dev/null
+++ b/src/ci/docker/scripts/shared.sh
@@ -0,0 +1,42 @@
+#!/bin/false
+# shellcheck shell=bash
+
+# This file is intended to be sourced with `. shared.sh` or
+# `source shared.sh`, hence the invalid shebang and not being
+# marked as an executable file in git.
+
+function hide_output {
+  { set +x; } 2>/dev/null
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  "$@" &> /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  set -x
+}
+
+# See https://unix.stackexchange.com/questions/82598
+# Duplicated in src/ci/shared.sh
+function retry {
+  echo "Attempting with retry:" "$@"
+  local n=1
+  local max=5
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        sleep $n  # don't retry immediately
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+      else
+        echo "The command has failed after $n attempts."
+        return 1
+      fi
+    }
+  done
+}
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 5b9c90273a3..800d8920951 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -470,6 +470,9 @@ jobs:
           - name: x86_64-gnu-aux
             <<: *job-linux-4c
 
+          - name: x86_64-gnu-integration
+            <<: *job-linux-16c
+
           - name: x86_64-gnu-debug
             <<: *job-linux-8c
 
@@ -730,6 +733,7 @@ jobs:
               CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
 
+
   master:
     name: master
     runs-on: ubuntu-latest
diff --git a/src/ci/shared.sh b/src/ci/shared.sh
index 8a88c56194c..720394af249 100644
--- a/src/ci/shared.sh
+++ b/src/ci/shared.sh
@@ -8,7 +8,7 @@
 export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc"
 
 # See https://unix.stackexchange.com/questions/82598
-# Duplicated in docker/dist-various-2/shared.sh
+# Duplicated in docker/scripts/shared.sh
 function retry {
   echo "Attempting with retry:" "$@"
   local n=1
-- 
cgit 1.4.1-3-g733a5


From ffb4c08a8117c8008ff696b6074cdf48889de211 Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Thu, 7 Dec 2023 18:20:27 +0100
Subject: implement and use `NormalizesTo`

---
 compiler/rustc_infer/src/infer/combine.rs          |  66 +-
 compiler/rustc_middle/src/ty/print/pretty.rs       |   8 +-
 .../src/solve/alias_relate.rs                      |   8 +-
 .../src/solve/assembly/mod.rs                      |   8 +-
 .../src/solve/canonicalize.rs                      |   4 +-
 .../src/solve/eval_ctxt/mod.rs                     |  20 +-
 compiler/rustc_trait_selection/src/solve/mod.rs    |  11 +-
 .../rustc_trait_selection/src/solve/normalize.rs   |   6 +-
 .../src/solve/normalizes_to/inherent.rs            |  50 ++
 .../src/solve/normalizes_to/mod.rs                 | 670 ++++++++++++++++++++
 .../src/solve/normalizes_to/opaques.rs             | 106 ++++
 .../src/solve/normalizes_to/weak_types.rs          |  34 ++
 .../src/solve/project_goals.rs                     |  23 +
 .../src/solve/project_goals/inherent_projection.rs |  50 --
 .../src/solve/project_goals/mod.rs                 | 676 ---------------------
 .../src/solve/project_goals/opaques.rs             | 106 ----
 .../src/solve/project_goals/weak_types.rs          |  34 --
 .../src/traits/structural_normalize.rs             |   8 +-
 compiler/rustc_type_ir/src/predicate_kind.rs       |  21 +-
 tests/ui/traits/new-solver/alias-bound-unsound.rs  |   2 +-
 .../traits/new-solver/alias-bound-unsound.stderr   |   2 +-
 .../generalize-proj-new-universe-index-2.rs        |   2 +
 .../occurs-check-nested-alias.next.stderr          |  10 +-
 .../generalize/occurs-check-nested-alias.rs        |   3 +-
 .../new-solver/specialization-transmute.stderr     |   6 +-
 25 files changed, 972 insertions(+), 962 deletions(-)
 create mode 100644 compiler/rustc_trait_selection/src/solve/normalizes_to/inherent.rs
 create mode 100644 compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
 create mode 100644 compiler/rustc_trait_selection/src/solve/normalizes_to/opaques.rs
 create mode 100644 compiler/rustc_trait_selection/src/solve/normalizes_to/weak_types.rs
 create mode 100644 compiler/rustc_trait_selection/src/solve/project_goals.rs
 delete mode 100644 compiler/rustc_trait_selection/src/solve/project_goals/inherent_projection.rs
 delete mode 100644 compiler/rustc_trait_selection/src/solve/project_goals/mod.rs
 delete mode 100644 compiler/rustc_trait_selection/src/solve/project_goals/opaques.rs
 delete mode 100644 compiler/rustc_trait_selection/src/solve/project_goals/weak_types.rs

diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs
index dfae279324f..cfd53fc71eb 100644
--- a/compiler/rustc_infer/src/infer/combine.rs
+++ b/compiler/rustc_infer/src/infer/combine.rs
@@ -34,8 +34,8 @@ use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue, EffectVa
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::relate::{RelateResult, TypeRelation};
-use rustc_middle::ty::TyVar;
 use rustc_middle::ty::{self, InferConst, ToPredicate, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::{AliasRelationDirection, TyVar};
 use rustc_middle::ty::{IntType, UintType};
 use rustc_span::DUMMY_SP;
 
@@ -490,31 +490,47 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
             // cyclic type. We instead delay the unification in case
             // the alias can be normalized to something which does not
             // mention `?0`.
-
-            // FIXME(-Ztrait-solver=next): replace this with `AliasRelate`
-            let &ty::Alias(kind, data) = a_ty.kind() else {
-                bug!("generalization should only result in infer vars for aliases");
-            };
-            if !self.infcx.next_trait_solver() {
-                // The old solver only accepts projection predicates for associated types.
-                match kind {
-                    ty::AliasKind::Projection => {}
-                    ty::AliasKind::Inherent | ty::AliasKind::Weak | ty::AliasKind::Opaque => {
-                        return Err(TypeError::CyclicTy(a_ty));
+            if self.infcx.next_trait_solver() {
+                let (lhs, rhs, direction) = match ambient_variance {
+                    ty::Variance::Invariant => {
+                        (a_ty.into(), b_ty.into(), AliasRelationDirection::Equate)
+                    }
+                    ty::Variance::Covariant => {
+                        (a_ty.into(), b_ty.into(), AliasRelationDirection::Subtype)
+                    }
+                    ty::Variance::Contravariant => {
+                        (b_ty.into(), a_ty.into(), AliasRelationDirection::Subtype)
                     }
+                    ty::Variance::Bivariant => unreachable!("bivariant generalization"),
+                };
+                self.obligations.push(Obligation::new(
+                    self.tcx(),
+                    self.trace.cause.clone(),
+                    self.param_env,
+                    ty::PredicateKind::AliasRelate(lhs, rhs, direction),
+                ));
+            } else {
+                match a_ty.kind() {
+                    &ty::Alias(ty::AliasKind::Projection, data) => {
+                        // FIXME: This does not handle subtyping correctly, we should switch to
+                        // alias-relate in the new solver and could instead create a new inference
+                        // variable for `a_ty`, emitting `Projection(a_ty, a_infer)` and
+                        // `a_infer <: b_ty`.
+                        self.obligations.push(Obligation::new(
+                            self.tcx(),
+                            self.trace.cause.clone(),
+                            self.param_env,
+                            ty::ProjectionPredicate { projection_ty: data, term: b_ty.into() },
+                        ))
+                    }
+                    // The old solver only accepts projection predicates for associated types.
+                    ty::Alias(
+                        ty::AliasKind::Inherent | ty::AliasKind::Weak | ty::AliasKind::Opaque,
+                        _,
+                    ) => return Err(TypeError::CyclicTy(a_ty)),
+                    _ => bug!("generalizated `{a_ty:?} to infer, not an alias"),
                 }
             }
-
-            // FIXME: This does not handle subtyping correctly, we should switch to
-            // alias-relate in the new solver and could instead create a new inference
-            // variable for `a_ty`, emitting `Projection(a_ty, a_infer)` and
-            // `a_infer <: b_ty`.
-            self.obligations.push(Obligation::new(
-                self.tcx(),
-                self.trace.cause.clone(),
-                self.param_env,
-                ty::ProjectionPredicate { projection_ty: data, term: b_ty.into() },
-            ))
         } else {
             match ambient_variance {
                 ty::Variance::Invariant => self.equate(a_is_expected).relate(a_ty, b_ty),
@@ -525,9 +541,7 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
                     a_ty,
                     b_ty,
                 ),
-                ty::Variance::Bivariant => {
-                    unreachable!("no code should be generalizing bivariantly (currently)")
-                }
+                ty::Variance::Bivariant => unreachable!("bivariant generalization"),
             }?;
         }
 
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index faf2a854f69..57253fa7c59 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -2814,7 +2814,7 @@ define_print! {
                 p!("the constant `", print(c1), "` equals `", print(c2), "`")
             }
             ty::PredicateKind::Ambiguous => p!("ambiguous"),
-            ty::PredicateKind::NormalizesTo(data) => p!(print(data.alias), " normalizes-to ", print(data.term)),
+            ty::PredicateKind::NormalizesTo(data) => p!(print(data)),
             ty::PredicateKind::AliasRelate(t1, t2, dir) => p!(print(t1), write(" {} ", dir), print(t2)),
         }
     }
@@ -2946,6 +2946,12 @@ define_print_and_forward_display! {
         p!(print(self.term))
     }
 
+    ty::NormalizesTo<'tcx> {
+        p!(print(self.alias), " normalizes-to ");
+        cx.reset_type_limit();
+        p!(print(self.term))
+    }
+
     ty::Term<'tcx> {
       match self.unpack() {
         ty::TermKind::Ty(ty) => p!(print(ty)),
diff --git a/compiler/rustc_trait_selection/src/solve/alias_relate.rs b/compiler/rustc_trait_selection/src/solve/alias_relate.rs
index 739bbe929b3..2e99854ddc6 100644
--- a/compiler/rustc_trait_selection/src/solve/alias_relate.rs
+++ b/compiler/rustc_trait_selection/src/solve/alias_relate.rs
@@ -92,7 +92,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                     self.add_goal(Goal::new(
                         self.tcx(),
                         param_env,
-                        ty::ProjectionPredicate { projection_ty: alias, term },
+                        ty::NormalizesTo { alias, term },
                     ));
                     self.try_evaluate_added_goals()?;
                     Ok(Some(self.resolve_vars_if_possible(term)))
@@ -109,11 +109,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         opaque: ty::AliasTy<'tcx>,
         term: ty::Term<'tcx>,
     ) -> QueryResult<'tcx> {
-        self.add_goal(Goal::new(
-            self.tcx(),
-            param_env,
-            ty::ProjectionPredicate { projection_ty: opaque, term },
-        ));
+        self.add_goal(Goal::new(self.tcx(), param_env, ty::NormalizesTo { alias: opaque, term }));
         self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
index b6861d258d1..201fade5ad7 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
@@ -352,15 +352,13 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         num_steps: usize,
     ) {
         let tcx = self.tcx();
-        let &ty::Alias(_, projection_ty) = goal.predicate.self_ty().kind() else { return };
+        let &ty::Alias(_, alias) = goal.predicate.self_ty().kind() else { return };
 
         candidates.extend(self.probe(|_| ProbeKind::NormalizedSelfTyAssembly).enter(|ecx| {
             if tcx.recursion_limit().value_within_limit(num_steps) {
                 let normalized_ty = ecx.next_ty_infer();
-                let normalizes_to_goal = goal.with(
-                    tcx,
-                    ty::ProjectionPredicate { projection_ty, term: normalized_ty.into() },
-                );
+                let normalizes_to_goal =
+                    goal.with(tcx, ty::NormalizesTo { alias, term: normalized_ty.into() });
                 ecx.add_goal(normalizes_to_goal);
                 if let Err(NoSolution) = ecx.try_evaluate_added_goals() {
                     debug!("self type normalization failed");
diff --git a/compiler/rustc_trait_selection/src/solve/canonicalize.rs b/compiler/rustc_trait_selection/src/solve/canonicalize.rs
index 004dc45263c..469d1f5a9cb 100644
--- a/compiler/rustc_trait_selection/src/solve/canonicalize.rs
+++ b/compiler/rustc_trait_selection/src/solve/canonicalize.rs
@@ -69,8 +69,8 @@ impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
         };
 
         let value = value.fold_with(&mut canonicalizer);
-        assert!(!value.has_infer());
-        assert!(!value.has_placeholders());
+        assert!(!value.has_infer(), "unexpected infer in {value:?}");
+        assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
 
         let (max_universe, variables) = canonicalizer.finalize();
 
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
index 348dfdf725f..b3e7a63c972 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
@@ -103,7 +103,7 @@ pub(super) struct NestedGoals<'tcx> {
     /// with a fresh inference variable when we evaluate this goal. That can result
     /// in a trait solver cycle. This would currently result in overflow but can be
     /// can be unsound with more powerful coinduction in the future.
-    pub(super) normalizes_to_hack_goal: Option<Goal<'tcx, ty::ProjectionPredicate<'tcx>>>,
+    pub(super) normalizes_to_hack_goal: Option<Goal<'tcx, ty::NormalizesTo<'tcx>>>,
     /// The rest of the goals which have not yet processed or remain ambiguous.
     pub(super) goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
 }
@@ -423,7 +423,9 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
                 ty::PredicateKind::ConstEquate(_, _) => {
                     bug!("ConstEquate should not be emitted when `-Ztrait-solver=next` is active")
                 }
-                ty::PredicateKind::NormalizesTo(_) => unimplemented!(),
+                ty::PredicateKind::NormalizesTo(predicate) => {
+                    self.compute_normalizes_to_goal(Goal { param_env, predicate })
+                }
                 ty::PredicateKind::AliasRelate(lhs, rhs, direction) => self
                     .compute_alias_relate_goal(Goal {
                         param_env,
@@ -493,10 +495,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             let unconstrained_rhs = self.next_term_infer_of_kind(goal.predicate.term);
             let unconstrained_goal = goal.with(
                 tcx,
-                ty::ProjectionPredicate {
-                    projection_ty: goal.predicate.projection_ty,
-                    term: unconstrained_rhs,
-                },
+                ty::NormalizesTo { alias: goal.predicate.alias, term: unconstrained_rhs },
             );
 
             let (_, certainty, instantiate_goals) = self.evaluate_goal(
@@ -518,9 +517,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             // looking at the "has changed" return from evaluate_goal,
             // because we expect the `unconstrained_rhs` part of the predicate
             // to have changed -- that means we actually normalized successfully!
-            if goal.predicate.projection_ty
-                != self.resolve_vars_if_possible(goal.predicate.projection_ty)
-            {
+            if goal.predicate.alias != self.resolve_vars_if_possible(goal.predicate.alias) {
                 unchanged_certainty = None;
             }
 
@@ -590,9 +587,10 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ///
     /// This is the case if the `term` is an inference variable in the innermost universe
     /// and does not occur in any other part of the predicate.
+    #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn term_is_fully_unconstrained(
         &self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
     ) -> bool {
         let term_is_infer = match goal.predicate.term.unpack() {
             ty::TermKind::Ty(ty) => {
@@ -656,7 +654,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         let mut visitor = ContainsTerm { infcx: self.infcx, term: goal.predicate.term };
 
         term_is_infer
-            && goal.predicate.projection_ty.visit_with(&mut visitor).is_continue()
+            && goal.predicate.alias.visit_with(&mut visitor).is_continue()
             && goal.param_env.visit_with(&mut visitor).is_continue()
     }
 
diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs
index bf3b72caeb4..cf3f94e26e4 100644
--- a/compiler/rustc_trait_selection/src/solve/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/mod.rs
@@ -35,6 +35,7 @@ mod eval_ctxt;
 mod fulfill;
 pub mod inspect;
 mod normalize;
+mod normalizes_to;
 mod project_goals;
 mod search_graph;
 mod trait_goals;
@@ -216,7 +217,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     #[instrument(level = "debug", skip(self))]
-    fn set_normalizes_to_hack_goal(&mut self, goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>) {
+    fn set_normalizes_to_hack_goal(&mut self, goal: Goal<'tcx, ty::NormalizesTo<'tcx>>) {
         assert!(
             self.nested_goals.normalizes_to_hack_goal.is_none(),
             "attempted to set the projection eq hack goal when one already exists"
@@ -310,17 +311,17 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             return None;
         }
 
-        let ty::Alias(kind, projection_ty) = *ty.kind() else {
+        let ty::Alias(kind, alias) = *ty.kind() else {
             return Some(ty);
         };
 
         // We do no always define opaque types eagerly to allow non-defining uses in the defining scope.
         if let (DefineOpaqueTypes::No, ty::AliasKind::Opaque) = (define_opaque_types, kind) {
-            if let Some(def_id) = projection_ty.def_id.as_local() {
+            if let Some(def_id) = alias.def_id.as_local() {
                 if self
                     .unify_existing_opaque_tys(
                         param_env,
-                        OpaqueTypeKey { def_id, args: projection_ty.args },
+                        OpaqueTypeKey { def_id, args: alias.args },
                         self.next_ty_infer(),
                     )
                     .is_empty()
@@ -335,7 +336,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             let normalizes_to_goal = Goal::new(
                 this.tcx(),
                 param_env,
-                ty::ProjectionPredicate { projection_ty, term: normalized_ty.into() },
+                ty::NormalizesTo { alias, term: normalized_ty.into() },
             );
             this.add_goal(normalizes_to_goal);
             this.try_evaluate_added_goals()?;
diff --git a/compiler/rustc_trait_selection/src/solve/normalize.rs b/compiler/rustc_trait_selection/src/solve/normalize.rs
index ea512ba5fa7..1e495b4d979 100644
--- a/compiler/rustc_trait_selection/src/solve/normalize.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalize.rs
@@ -76,7 +76,7 @@ impl<'tcx> NormalizationFolder<'_, 'tcx> {
             tcx,
             self.at.cause.clone(),
             self.at.param_env,
-            ty::ProjectionPredicate { projection_ty: alias, term: new_infer_ty.into() },
+            ty::NormalizesTo { alias, term: new_infer_ty.into() },
         );
 
         // Do not emit an error if normalization is known to fail but instead
@@ -129,8 +129,8 @@ impl<'tcx> NormalizationFolder<'_, 'tcx> {
             tcx,
             self.at.cause.clone(),
             self.at.param_env,
-            ty::ProjectionPredicate {
-                projection_ty: AliasTy::new(tcx, uv.def, uv.args),
+            ty::NormalizesTo {
+                alias: AliasTy::new(tcx, uv.def, uv.args),
                 term: new_infer_ct.into(),
             },
         );
diff --git a/compiler/rustc_trait_selection/src/solve/normalizes_to/inherent.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/inherent.rs
new file mode 100644
index 00000000000..c3b8ae9a943
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/inherent.rs
@@ -0,0 +1,50 @@
+//! Computes a normalizes-to (projection) goal for inherent associated types,
+//! `#![feature(inherent_associated_type)]`. Since astconv already determines
+//! which impl the IAT is being projected from, we just:
+//! 1. instantiate substs,
+//! 2. equate the self type, and
+//! 3. instantiate and register where clauses.
+use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
+use rustc_middle::ty;
+
+use super::EvalCtxt;
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    pub(super) fn normalize_inherent_associated_type(
+        &mut self,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        let tcx = self.tcx();
+        let inherent = goal.predicate.alias;
+        let expected = goal.predicate.term.ty().expect("inherent consts are treated separately");
+
+        let impl_def_id = tcx.parent(inherent.def_id);
+        let impl_substs = self.fresh_args_for_item(impl_def_id);
+
+        // Equate impl header and add impl where clauses
+        self.eq(
+            goal.param_env,
+            inherent.self_ty(),
+            tcx.type_of(impl_def_id).instantiate(tcx, impl_substs),
+        )?;
+
+        // Equate IAT with the RHS of the project goal
+        let inherent_substs = inherent.rebase_inherent_args_onto_impl(impl_substs, tcx);
+        self.eq(
+            goal.param_env,
+            expected,
+            tcx.type_of(inherent.def_id).instantiate(tcx, inherent_substs),
+        )
+        .expect("expected goal term to be fully unconstrained");
+
+        // Check both where clauses on the impl and IAT
+        self.add_goals(
+            tcx.predicates_of(inherent.def_id)
+                .instantiate(tcx, inherent_substs)
+                .into_iter()
+                .map(|(pred, _)| goal.with(tcx, pred)),
+        );
+
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
new file mode 100644
index 00000000000..867a520915f
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
@@ -0,0 +1,670 @@
+use crate::traits::{check_args_compatible, specialization_graph};
+
+use super::assembly::{self, structural_traits, Candidate};
+use super::EvalCtxt;
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::DefId;
+use rustc_hir::LangItem;
+use rustc_infer::traits::query::NoSolution;
+use rustc_infer::traits::specialization_graph::LeafDef;
+use rustc_infer::traits::Reveal;
+use rustc_middle::traits::solve::{
+    CandidateSource, CanonicalResponse, Certainty, Goal, QueryResult,
+};
+use rustc_middle::traits::BuiltinImplSource;
+use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_middle::ty::NormalizesTo;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{ToPredicate, TypeVisitableExt};
+use rustc_span::{sym, ErrorGuaranteed, DUMMY_SP};
+
+mod inherent;
+mod opaques;
+mod weak_types;
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    #[instrument(level = "debug", skip(self), ret)]
+    pub(super) fn compute_normalizes_to_goal(
+        &mut self,
+        goal: Goal<'tcx, NormalizesTo<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        let def_id = goal.predicate.def_id();
+        match self.tcx().def_kind(def_id) {
+            DefKind::AssocTy | DefKind::AssocConst => {
+                // To only compute normalization once for each projection we only
+                // assemble normalization candidates if the expected term is an
+                // unconstrained inference variable.
+                //
+                // Why: For better cache hits, since if we have an unconstrained RHS then
+                // there are only as many cache keys as there are (canonicalized) alias
+                // types in each normalizes-to goal. This also weakens inference in a
+                // forwards-compatible way so we don't use the value of the RHS term to
+                // affect candidate assembly for projections.
+                //
+                // E.g. for `<T as Trait>::Assoc == u32` we recursively compute the goal
+                // `exists<U> <T as Trait>::Assoc == U` and then take the resulting type for
+                // `U` and equate it with `u32`. This means that we don't need a separate
+                // projection cache in the solver, since we're piggybacking off of regular
+                // goal caching.
+                if self.term_is_fully_unconstrained(goal) {
+                    match self.tcx().associated_item(def_id).container {
+                        ty::AssocItemContainer::TraitContainer => {
+                            let candidates = self.assemble_and_evaluate_candidates(goal);
+                            self.merge_candidates(candidates)
+                        }
+                        ty::AssocItemContainer::ImplContainer => {
+                            self.normalize_inherent_associated_type(goal)
+                        }
+                    }
+                } else {
+                    self.set_normalizes_to_hack_goal(goal);
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                }
+            }
+            DefKind::AnonConst => self.normalize_anon_const(goal),
+            DefKind::OpaqueTy => self.normalize_opaque_type(goal),
+            DefKind::TyAlias => self.normalize_weak_type(goal),
+            kind => bug!("unknown DefKind {} in projection goal: {goal:#?}", kind.descr(def_id)),
+        }
+    }
+
+    #[instrument(level = "debug", skip(self), ret)]
+    fn normalize_anon_const(
+        &mut self,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        if let Some(normalized_const) = self.try_const_eval_resolve(
+            goal.param_env,
+            ty::UnevaluatedConst::new(goal.predicate.alias.def_id, goal.predicate.alias.args),
+            self.tcx()
+                .type_of(goal.predicate.alias.def_id)
+                .no_bound_vars()
+                .expect("const ty should not rely on other generics"),
+        ) {
+            self.eq(goal.param_env, normalized_const, goal.predicate.term.ct().unwrap())?;
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        } else {
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+        }
+    }
+}
+
+impl<'tcx> assembly::GoalKind<'tcx> for NormalizesTo<'tcx> {
+    fn self_ty(self) -> Ty<'tcx> {
+        self.self_ty()
+    }
+
+    fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx> {
+        self.alias.trait_ref(tcx)
+    }
+
+    fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
+        self.with_self_ty(tcx, self_ty)
+    }
+
+    fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId {
+        self.trait_def_id(tcx)
+    }
+
+    fn probe_and_match_goal_against_assumption(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        assumption: ty::Clause<'tcx>,
+        then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>,
+    ) -> QueryResult<'tcx> {
+        if let Some(projection_pred) = assumption.as_projection_clause() {
+            if projection_pred.projection_def_id() == goal.predicate.def_id() {
+                let tcx = ecx.tcx();
+                ecx.probe_misc_candidate("assumption").enter(|ecx| {
+                    let assumption_projection_pred =
+                        ecx.instantiate_binder_with_infer(projection_pred);
+                    ecx.eq(
+                        goal.param_env,
+                        goal.predicate.alias,
+                        assumption_projection_pred.projection_ty,
+                    )?;
+                    ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)
+                        .expect("expected goal term to be fully unconstrained");
+
+                    // Add GAT where clauses from the trait's definition
+                    ecx.add_goals(
+                        tcx.predicates_of(goal.predicate.def_id())
+                            .instantiate_own(tcx, goal.predicate.alias.args)
+                            .map(|(pred, _)| goal.with(tcx, pred)),
+                    );
+
+                    then(ecx)
+                })
+            } else {
+                Err(NoSolution)
+            }
+        } else {
+            Err(NoSolution)
+        }
+    }
+
+    fn consider_impl_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, NormalizesTo<'tcx>>,
+        impl_def_id: DefId,
+    ) -> Result<Candidate<'tcx>, NoSolution> {
+        let tcx = ecx.tcx();
+
+        let goal_trait_ref = goal.predicate.alias.trait_ref(tcx);
+        let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
+        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::ForLookup };
+        if !drcx.args_may_unify(goal_trait_ref.args, impl_trait_ref.skip_binder().args) {
+            return Err(NoSolution);
+        }
+
+        ecx.probe_trait_candidate(CandidateSource::Impl(impl_def_id)).enter(|ecx| {
+            let impl_args = ecx.fresh_args_for_item(impl_def_id);
+            let impl_trait_ref = impl_trait_ref.instantiate(tcx, impl_args);
+
+            ecx.eq(goal.param_env, goal_trait_ref, impl_trait_ref)?;
+
+            let where_clause_bounds = tcx
+                .predicates_of(impl_def_id)
+                .instantiate(tcx, impl_args)
+                .predicates
+                .into_iter()
+                .map(|pred| goal.with(tcx, pred));
+            ecx.add_goals(where_clause_bounds);
+
+            // Add GAT where clauses from the trait's definition
+            ecx.add_goals(
+                tcx.predicates_of(goal.predicate.def_id())
+                    .instantiate_own(tcx, goal.predicate.alias.args)
+                    .map(|(pred, _)| goal.with(tcx, pred)),
+            );
+
+            // In case the associated item is hidden due to specialization, we have to
+            // return ambiguity this would otherwise be incomplete, resulting in
+            // unsoundness during coherence (#105782).
+            let Some(assoc_def) = fetch_eligible_assoc_item_def(
+                ecx,
+                goal.param_env,
+                goal_trait_ref,
+                goal.predicate.def_id(),
+                impl_def_id,
+            )?
+            else {
+                return ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
+            };
+
+            let error_response = |ecx: &mut EvalCtxt<'_, 'tcx>, reason| {
+                let guar = tcx.sess.span_delayed_bug(tcx.def_span(assoc_def.item.def_id), reason);
+                let error_term = match assoc_def.item.kind {
+                    ty::AssocKind::Const => ty::Const::new_error(
+                        tcx,
+                        guar,
+                        tcx.type_of(goal.predicate.def_id())
+                            .instantiate(tcx, goal.predicate.alias.args),
+                    )
+                    .into(),
+                    ty::AssocKind::Type => Ty::new_error(tcx, guar).into(),
+                    ty::AssocKind::Fn => unreachable!(),
+                };
+                ecx.eq(goal.param_env, goal.predicate.term, error_term)
+                    .expect("expected goal term to be fully unconstrained");
+                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            };
+
+            if !assoc_def.item.defaultness(tcx).has_value() {
+                return error_response(ecx, "missing value for assoc item in impl");
+            }
+
+            // Getting the right args here is complex, e.g. given:
+            // - a goal `<Vec<u32> as Trait<i32>>::Assoc<u64>`
+            // - the applicable impl `impl<T> Trait<i32> for Vec<T>`
+            // - and the impl which defines `Assoc` being `impl<T, U> Trait<U> for Vec<T>`
+            //
+            // We first rebase the goal args onto the impl, going from `[Vec<u32>, i32, u64]`
+            // to `[u32, u64]`.
+            //
+            // And then map these args to the args of the defining impl of `Assoc`, going
+            // from `[u32, u64]` to `[u32, i32, u64]`.
+            let impl_args_with_gat =
+                goal.predicate.alias.args.rebase_onto(tcx, goal_trait_ref.def_id, impl_args);
+            let args = ecx.translate_args(
+                goal.param_env,
+                impl_def_id,
+                impl_args_with_gat,
+                assoc_def.defining_node,
+            );
+
+            if !check_args_compatible(tcx, assoc_def.item, args) {
+                return error_response(
+                    ecx,
+                    "associated item has mismatched generic item arguments",
+                );
+            }
+
+            // Finally we construct the actual value of the associated type.
+            let term = match assoc_def.item.kind {
+                ty::AssocKind::Type => tcx.type_of(assoc_def.item.def_id).map_bound(|ty| ty.into()),
+                ty::AssocKind::Const => {
+                    if tcx.features().associated_const_equality {
+                        bug!("associated const projection is not supported yet")
+                    } else {
+                        ty::EarlyBinder::bind(
+                            ty::Const::new_error_with_message(
+                                tcx,
+                                tcx.type_of(assoc_def.item.def_id).instantiate_identity(),
+                                DUMMY_SP,
+                                "associated const projection is not supported yet",
+                            )
+                            .into(),
+                        )
+                    }
+                }
+                ty::AssocKind::Fn => unreachable!("we should never project to a fn"),
+            };
+
+            ecx.eq(goal.param_env, goal.predicate.term, term.instantiate(tcx, args))
+                .expect("expected goal term to be fully unconstrained");
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
+
+    /// Fail to normalize if the predicate contains an error, alternatively, we could normalize to `ty::Error`
+    /// and succeed. Can experiment with this to figure out what results in better error messages.
+    fn consider_error_guaranteed_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        _guar: ErrorGuaranteed,
+    ) -> QueryResult<'tcx> {
+        Err(NoSolution)
+    }
+
+    fn consider_auto_trait_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        ecx.tcx().sess.span_delayed_bug(
+            ecx.tcx().def_span(goal.predicate.def_id()),
+            "associated types not allowed on auto traits",
+        );
+        Err(NoSolution)
+    }
+
+    fn consider_trait_alias_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("trait aliases do not have associated types: {:?}", goal);
+    }
+
+    fn consider_builtin_sized_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`Sized` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_copy_clone_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`Copy`/`Clone` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_pointer_like_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`PointerLike` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_fn_ptr_trait_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`FnPtr` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_fn_trait_candidates(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        goal_kind: ty::ClosureKind,
+    ) -> QueryResult<'tcx> {
+        let tcx = ecx.tcx();
+        let tupled_inputs_and_output =
+            match structural_traits::extract_tupled_inputs_and_output_from_callable(
+                tcx,
+                goal.predicate.self_ty(),
+                goal_kind,
+            )? {
+                Some(tupled_inputs_and_output) => tupled_inputs_and_output,
+                None => {
+                    return ecx
+                        .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
+                }
+            };
+        let output_is_sized_pred = tupled_inputs_and_output.map_bound(|(_, output)| {
+            ty::TraitRef::from_lang_item(tcx, LangItem::Sized, DUMMY_SP, [output])
+        });
+
+        let pred = tupled_inputs_and_output
+            .map_bound(|(inputs, output)| ty::ProjectionPredicate {
+                projection_ty: ty::AliasTy::new(
+                    tcx,
+                    goal.predicate.def_id(),
+                    [goal.predicate.self_ty(), inputs],
+                ),
+                term: output.into(),
+            })
+            .to_predicate(tcx);
+
+        // A built-in `Fn` impl only holds if the output is sized.
+        // (FIXME: technically we only need to check this if the type is a fn ptr...)
+        Self::consider_implied_clause(ecx, goal, pred, [goal.with(tcx, output_is_sized_pred)])
+    }
+
+    fn consider_builtin_tuple_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`Tuple` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_pointee_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let tcx = ecx.tcx();
+        ecx.probe_misc_candidate("builtin pointee").enter(|ecx| {
+            let metadata_ty = match goal.predicate.self_ty().kind() {
+                ty::Bool
+                | ty::Char
+                | ty::Int(..)
+                | ty::Uint(..)
+                | ty::Float(..)
+                | ty::Array(..)
+                | ty::RawPtr(..)
+                | ty::Ref(..)
+                | ty::FnDef(..)
+                | ty::FnPtr(..)
+                | ty::Closure(..)
+                | ty::Infer(ty::IntVar(..) | ty::FloatVar(..))
+                | ty::Coroutine(..)
+                | ty::CoroutineWitness(..)
+                | ty::Never
+                | ty::Foreign(..) => tcx.types.unit,
+
+                ty::Error(e) => Ty::new_error(tcx, *e),
+
+                ty::Str | ty::Slice(_) => tcx.types.usize,
+
+                ty::Dynamic(_, _, _) => {
+                    let dyn_metadata = tcx.require_lang_item(LangItem::DynMetadata, None);
+                    tcx.type_of(dyn_metadata)
+                        .instantiate(tcx, &[ty::GenericArg::from(goal.predicate.self_ty())])
+                }
+
+                ty::Alias(_, _) | ty::Param(_) | ty::Placeholder(..) => {
+                    // FIXME(ptr_metadata): It would also be possible to return a `Ok(Ambig)` with no constraints.
+                    let sized_predicate = ty::TraitRef::from_lang_item(
+                        tcx,
+                        LangItem::Sized,
+                        DUMMY_SP,
+                        [ty::GenericArg::from(goal.predicate.self_ty())],
+                    );
+                    ecx.add_goal(goal.with(tcx, sized_predicate));
+                    tcx.types.unit
+                }
+
+                ty::Adt(def, args) if def.is_struct() => match def.non_enum_variant().tail_opt() {
+                    None => tcx.types.unit,
+                    Some(field_def) => {
+                        let self_ty = field_def.ty(tcx, args);
+                        ecx.add_goal(goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)));
+                        return ecx
+                            .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
+                    }
+                },
+                ty::Adt(_, _) => tcx.types.unit,
+
+                ty::Tuple(elements) => match elements.last() {
+                    None => tcx.types.unit,
+                    Some(&self_ty) => {
+                        ecx.add_goal(goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)));
+                        return ecx
+                            .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
+                    }
+                },
+
+                ty::Infer(
+                    ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_),
+                )
+                | ty::Bound(..) => bug!(
+                    "unexpected self ty `{:?}` when normalizing `<T as Pointee>::Metadata`",
+                    goal.predicate.self_ty()
+                ),
+            };
+
+            ecx.eq(goal.param_env, goal.predicate.term, metadata_ty.into())
+                .expect("expected goal term to be fully unconstrained");
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
+
+    fn consider_builtin_future_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not futures unless they come from `async` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_async(def_id) {
+            return Err(NoSolution);
+        }
+
+        let term = args.as_coroutine().return_ty().into();
+
+        Self::consider_implied_clause(
+            ecx,
+            goal,
+            ty::ProjectionPredicate {
+                projection_ty: ty::AliasTy::new(ecx.tcx(), goal.predicate.def_id(), [self_ty]),
+                term,
+            }
+            .to_predicate(tcx),
+            // Technically, we need to check that the future type is Sized,
+            // but that's already proven by the coroutine being WF.
+            [],
+        )
+    }
+
+    fn consider_builtin_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not Iterators unless they come from `gen` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_gen(def_id) {
+            return Err(NoSolution);
+        }
+
+        let term = args.as_coroutine().yield_ty().into();
+
+        Self::consider_implied_clause(
+            ecx,
+            goal,
+            ty::ProjectionPredicate {
+                projection_ty: ty::AliasTy::new(ecx.tcx(), goal.predicate.def_id(), [self_ty]),
+                term,
+            }
+            .to_predicate(tcx),
+            // Technically, we need to check that the iterator type is Sized,
+            // but that's already proven by the generator being WF.
+            [],
+        )
+    }
+
+    fn consider_builtin_coroutine_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // `async`-desugared coroutines do not implement the coroutine trait
+        let tcx = ecx.tcx();
+        if !tcx.is_general_coroutine(def_id) {
+            return Err(NoSolution);
+        }
+
+        let coroutine = args.as_coroutine();
+
+        let name = tcx.associated_item(goal.predicate.def_id()).name;
+        let term = if name == sym::Return {
+            coroutine.return_ty().into()
+        } else if name == sym::Yield {
+            coroutine.yield_ty().into()
+        } else {
+            bug!("unexpected associated item `<{self_ty} as Coroutine>::{name}`")
+        };
+
+        Self::consider_implied_clause(
+            ecx,
+            goal,
+            ty::ProjectionPredicate {
+                projection_ty: ty::AliasTy::new(
+                    ecx.tcx(),
+                    goal.predicate.def_id(),
+                    [self_ty, coroutine.resume_ty()],
+                ),
+                term,
+            }
+            .to_predicate(tcx),
+            // Technically, we need to check that the coroutine type is Sized,
+            // but that's already proven by the coroutine being WF.
+            [],
+        )
+    }
+
+    fn consider_unsize_to_dyn_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`Unsize` does not have an associated type: {:?}", goal)
+    }
+
+    fn consider_structural_builtin_unsize_candidates(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> Vec<(CanonicalResponse<'tcx>, BuiltinImplSource)> {
+        bug!("`Unsize` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_discriminant_kind_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let discriminant_ty = match *self_ty.kind() {
+            ty::Bool
+            | ty::Char
+            | ty::Int(..)
+            | ty::Uint(..)
+            | ty::Float(..)
+            | ty::Array(..)
+            | ty::RawPtr(..)
+            | ty::Ref(..)
+            | ty::FnDef(..)
+            | ty::FnPtr(..)
+            | ty::Closure(..)
+            | ty::Infer(ty::IntVar(..) | ty::FloatVar(..))
+            | ty::Coroutine(..)
+            | ty::CoroutineWitness(..)
+            | ty::Never
+            | ty::Foreign(..)
+            | ty::Adt(_, _)
+            | ty::Str
+            | ty::Slice(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Tuple(_)
+            | ty::Error(_) => self_ty.discriminant_ty(ecx.tcx()),
+
+            // We do not call `Ty::discriminant_ty` on alias, param, or placeholder
+            // types, which return `<self_ty as DiscriminantKind>::Discriminant`
+            // (or ICE in the case of placeholders). Projecting a type to itself
+            // is never really productive.
+            ty::Alias(_, _) | ty::Param(_) | ty::Placeholder(..) => {
+                return Err(NoSolution);
+            }
+
+            ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_))
+            | ty::Bound(..) => bug!(
+                "unexpected self ty `{:?}` when normalizing `<T as DiscriminantKind>::Discriminant`",
+                goal.predicate.self_ty()
+            ),
+        };
+
+        ecx.probe_misc_candidate("builtin discriminant kind").enter(|ecx| {
+            ecx.eq(goal.param_env, goal.predicate.term, discriminant_ty.into())
+                .expect("expected goal term to be fully unconstrained");
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
+
+    fn consider_builtin_destruct_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`Destruct` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_transmute_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`BikeshedIntrinsicFrom` does not have an associated type: {:?}", goal)
+    }
+}
+
+/// This behavior is also implemented in `rustc_ty_utils` and in the old `project` code.
+///
+/// FIXME: We should merge these 3 implementations as it's likely that they otherwise
+/// diverge.
+#[instrument(level = "debug", skip(ecx, param_env), ret)]
+fn fetch_eligible_assoc_item_def<'tcx>(
+    ecx: &EvalCtxt<'_, 'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    goal_trait_ref: ty::TraitRef<'tcx>,
+    trait_assoc_def_id: DefId,
+    impl_def_id: DefId,
+) -> Result<Option<LeafDef>, NoSolution> {
+    let node_item = specialization_graph::assoc_def(ecx.tcx(), impl_def_id, trait_assoc_def_id)
+        .map_err(|ErrorGuaranteed { .. }| NoSolution)?;
+
+    let eligible = if node_item.is_final() {
+        // Non-specializable items are always projectable.
+        true
+    } else {
+        // Only reveal a specializable default if we're past type-checking
+        // and the obligation is monomorphic, otherwise passes such as
+        // transmute checking and polymorphic MIR optimizations could
+        // get a result which isn't correct for all monomorphizations.
+        if param_env.reveal() == Reveal::All {
+            let poly_trait_ref = ecx.resolve_vars_if_possible(goal_trait_ref);
+            !poly_trait_ref.still_further_specializable()
+        } else {
+            debug!(?node_item.item.def_id, "not eligible due to default");
+            false
+        }
+    };
+
+    if eligible { Ok(Some(node_item)) } else { Ok(None) }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaques.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/opaques.rs
new file mode 100644
index 00000000000..b5d1aa06e4e
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/opaques.rs
@@ -0,0 +1,106 @@
+//! Computes a normalizes-to (projection) goal for opaque types. This goal
+//! behaves differently depending on the param-env's reveal mode and whether
+//! the opaque is in a defining scope.
+use rustc_middle::traits::query::NoSolution;
+use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
+use rustc_middle::traits::Reveal;
+use rustc_middle::ty;
+use rustc_middle::ty::util::NotUniqueParam;
+
+use crate::solve::{EvalCtxt, SolverMode};
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    pub(super) fn normalize_opaque_type(
+        &mut self,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        let tcx = self.tcx();
+        let opaque_ty = goal.predicate.alias;
+        let expected = goal.predicate.term.ty().expect("no such thing as an opaque const");
+
+        match (goal.param_env.reveal(), self.solver_mode()) {
+            (Reveal::UserFacing, SolverMode::Normal) => {
+                let Some(opaque_ty_def_id) = opaque_ty.def_id.as_local() else {
+                    return Err(NoSolution);
+                };
+                // FIXME: at some point we should call queries without defining
+                // new opaque types but having the existing opaque type definitions.
+                // This will require moving this below "Prefer opaques registered already".
+                if !self.can_define_opaque_ty(opaque_ty_def_id) {
+                    return Err(NoSolution);
+                }
+                // FIXME: This may have issues when the args contain aliases...
+                match self.tcx().uses_unique_placeholders_ignoring_regions(opaque_ty.args) {
+                    Err(NotUniqueParam::NotParam(param)) if param.is_non_region_infer() => {
+                        return self.evaluate_added_goals_and_make_canonical_response(
+                            Certainty::AMBIGUOUS,
+                        );
+                    }
+                    Err(_) => {
+                        return Err(NoSolution);
+                    }
+                    Ok(()) => {}
+                }
+                // Prefer opaques registered already.
+                let opaque_type_key =
+                    ty::OpaqueTypeKey { def_id: opaque_ty_def_id, args: opaque_ty.args };
+                // FIXME: This also unifies the previous hidden type with the expected.
+                //
+                // If that fails, we insert `expected` as a new hidden type instead of
+                // eagerly emitting an error.
+                let matches =
+                    self.unify_existing_opaque_tys(goal.param_env, opaque_type_key, expected);
+                if !matches.is_empty() {
+                    if let Some(response) = self.try_merge_responses(&matches) {
+                        return Ok(response);
+                    } else {
+                        return self.flounder(&matches);
+                    }
+                }
+
+                let expected = match self.try_normalize_ty(goal.param_env, expected) {
+                    Some(ty) => {
+                        if ty.is_ty_var() {
+                            return self.evaluate_added_goals_and_make_canonical_response(
+                                Certainty::AMBIGUOUS,
+                            );
+                        } else {
+                            ty
+                        }
+                    }
+                    None => {
+                        return self
+                            .evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
+                    }
+                };
+
+                // Otherwise, define a new opaque type
+                self.insert_hidden_type(opaque_type_key, goal.param_env, expected)?;
+                self.add_item_bounds_for_hidden_type(
+                    opaque_ty.def_id,
+                    opaque_ty.args,
+                    goal.param_env,
+                    expected,
+                );
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            }
+            (Reveal::UserFacing, SolverMode::Coherence) => {
+                // An impossible opaque type bound is the only way this goal will fail
+                // e.g. assigning `impl Copy := NotCopy`
+                self.add_item_bounds_for_hidden_type(
+                    opaque_ty.def_id,
+                    opaque_ty.args,
+                    goal.param_env,
+                    expected,
+                );
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+            }
+            (Reveal::All, _) => {
+                // FIXME: Add an assertion that opaque type storage is empty.
+                let actual = tcx.type_of(opaque_ty.def_id).instantiate(tcx, opaque_ty.args);
+                self.eq(goal.param_env, expected, actual)?;
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/normalizes_to/weak_types.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/weak_types.rs
new file mode 100644
index 00000000000..8d2bbec6d8b
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/weak_types.rs
@@ -0,0 +1,34 @@
+//! Computes a normalizes-to (projection) goal for inherent associated types,
+//! `#![feature(lazy_type_alias)]` and `#![feature(type_alias_impl_trait)]`.
+//!
+//! Since a weak alias is not ambiguous, this just computes the `type_of` of
+//! the alias and registers the where-clauses of the type alias.
+use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
+use rustc_middle::ty;
+
+use super::EvalCtxt;
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    pub(super) fn normalize_weak_type(
+        &mut self,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        let tcx = self.tcx();
+        let weak_ty = goal.predicate.alias;
+        let expected = goal.predicate.term.ty().expect("no such thing as a const alias");
+
+        let actual = tcx.type_of(weak_ty.def_id).instantiate(tcx, weak_ty.args);
+        self.eq(goal.param_env, expected, actual)?;
+
+        // Check where clauses
+        self.add_goals(
+            tcx.predicates_of(weak_ty.def_id)
+                .instantiate(tcx, weak_ty.args)
+                .predicates
+                .into_iter()
+                .map(|pred| goal.with(tcx, pred)),
+        );
+
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals.rs b/compiler/rustc_trait_selection/src/solve/project_goals.rs
new file mode 100644
index 00000000000..0b80969c307
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/project_goals.rs
@@ -0,0 +1,23 @@
+use super::EvalCtxt;
+use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
+use rustc_middle::ty::{self, ProjectionPredicate};
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    #[instrument(level = "debug", skip(self), ret)]
+    pub(super) fn compute_projection_goal(
+        &mut self,
+        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        match goal.predicate.term.unpack() {
+            ty::TermKind::Ty(term) => {
+                let alias = goal.predicate.projection_ty.to_ty(self.tcx());
+                self.eq(goal.param_env, alias, term)?;
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            }
+            // FIXME(associated_const_equality): actually do something here.
+            ty::TermKind::Const(_) => {
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/inherent_projection.rs b/compiler/rustc_trait_selection/src/solve/project_goals/inherent_projection.rs
deleted file mode 100644
index 28fe59b7f6a..00000000000
--- a/compiler/rustc_trait_selection/src/solve/project_goals/inherent_projection.rs
+++ /dev/null
@@ -1,50 +0,0 @@
-//! Computes a normalizes-to (projection) goal for inherent associated types,
-//! `#![feature(inherent_associated_type)]`. Since astconv already determines
-//! which impl the IAT is being projected from, we just:
-//! 1. instantiate substs,
-//! 2. equate the self type, and
-//! 3. instantiate and register where clauses.
-use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
-use rustc_middle::ty;
-
-use super::EvalCtxt;
-
-impl<'tcx> EvalCtxt<'_, 'tcx> {
-    pub(super) fn normalize_inherent_associated_type(
-        &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        let tcx = self.tcx();
-        let inherent = goal.predicate.projection_ty;
-        let expected = goal.predicate.term.ty().expect("inherent consts are treated separately");
-
-        let impl_def_id = tcx.parent(inherent.def_id);
-        let impl_substs = self.fresh_args_for_item(impl_def_id);
-
-        // Equate impl header and add impl where clauses
-        self.eq(
-            goal.param_env,
-            inherent.self_ty(),
-            tcx.type_of(impl_def_id).instantiate(tcx, impl_substs),
-        )?;
-
-        // Equate IAT with the RHS of the project goal
-        let inherent_substs = inherent.rebase_inherent_args_onto_impl(impl_substs, tcx);
-        self.eq(
-            goal.param_env,
-            expected,
-            tcx.type_of(inherent.def_id).instantiate(tcx, inherent_substs),
-        )
-        .expect("expected goal term to be fully unconstrained");
-
-        // Check both where clauses on the impl and IAT
-        self.add_goals(
-            tcx.predicates_of(inherent.def_id)
-                .instantiate(tcx, inherent_substs)
-                .into_iter()
-                .map(|(pred, _)| goal.with(tcx, pred)),
-        );
-
-        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs b/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs
deleted file mode 100644
index 03823569669..00000000000
--- a/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs
+++ /dev/null
@@ -1,676 +0,0 @@
-use crate::traits::{check_args_compatible, specialization_graph};
-
-use super::assembly::{self, structural_traits, Candidate};
-use super::EvalCtxt;
-use rustc_hir::def::DefKind;
-use rustc_hir::def_id::DefId;
-use rustc_hir::LangItem;
-use rustc_infer::traits::query::NoSolution;
-use rustc_infer::traits::specialization_graph::LeafDef;
-use rustc_infer::traits::Reveal;
-use rustc_middle::traits::solve::{
-    CandidateSource, CanonicalResponse, Certainty, Goal, QueryResult,
-};
-use rustc_middle::traits::BuiltinImplSource;
-use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
-use rustc_middle::ty::ProjectionPredicate;
-use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_middle::ty::{ToPredicate, TypeVisitableExt};
-use rustc_span::{sym, ErrorGuaranteed, DUMMY_SP};
-
-mod inherent_projection;
-mod opaques;
-mod weak_types;
-
-impl<'tcx> EvalCtxt<'_, 'tcx> {
-    #[instrument(level = "debug", skip(self), ret)]
-    pub(super) fn compute_projection_goal(
-        &mut self,
-        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        let def_id = goal.predicate.def_id();
-        match self.tcx().def_kind(def_id) {
-            DefKind::AssocTy | DefKind::AssocConst => {
-                // To only compute normalization once for each projection we only
-                // assemble normalization candidates if the expected term is an
-                // unconstrained inference variable.
-                //
-                // Why: For better cache hits, since if we have an unconstrained RHS then
-                // there are only as many cache keys as there are (canonicalized) alias
-                // types in each normalizes-to goal. This also weakens inference in a
-                // forwards-compatible way so we don't use the value of the RHS term to
-                // affect candidate assembly for projections.
-                //
-                // E.g. for `<T as Trait>::Assoc == u32` we recursively compute the goal
-                // `exists<U> <T as Trait>::Assoc == U` and then take the resulting type for
-                // `U` and equate it with `u32`. This means that we don't need a separate
-                // projection cache in the solver, since we're piggybacking off of regular
-                // goal caching.
-                if self.term_is_fully_unconstrained(goal) {
-                    match self.tcx().associated_item(def_id).container {
-                        ty::AssocItemContainer::TraitContainer => {
-                            let candidates = self.assemble_and_evaluate_candidates(goal);
-                            self.merge_candidates(candidates)
-                        }
-                        ty::AssocItemContainer::ImplContainer => {
-                            self.normalize_inherent_associated_type(goal)
-                        }
-                    }
-                } else {
-                    self.set_normalizes_to_hack_goal(goal);
-                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-                }
-            }
-            DefKind::AnonConst => self.normalize_anon_const(goal),
-            DefKind::OpaqueTy => self.normalize_opaque_type(goal),
-            DefKind::TyAlias => self.normalize_weak_type(goal),
-            kind => bug!("unknown DefKind {} in projection goal: {goal:#?}", kind.descr(def_id)),
-        }
-    }
-
-    #[instrument(level = "debug", skip(self), ret)]
-    fn normalize_anon_const(
-        &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        if let Some(normalized_const) = self.try_const_eval_resolve(
-            goal.param_env,
-            ty::UnevaluatedConst::new(
-                goal.predicate.projection_ty.def_id,
-                goal.predicate.projection_ty.args,
-            ),
-            self.tcx()
-                .type_of(goal.predicate.projection_ty.def_id)
-                .no_bound_vars()
-                .expect("const ty should not rely on other generics"),
-        ) {
-            self.eq(goal.param_env, normalized_const, goal.predicate.term.ct().unwrap())?;
-            self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        } else {
-            self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
-        }
-    }
-}
-
-impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
-    fn self_ty(self) -> Ty<'tcx> {
-        self.self_ty()
-    }
-
-    fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx> {
-        self.projection_ty.trait_ref(tcx)
-    }
-
-    fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
-        self.with_self_ty(tcx, self_ty)
-    }
-
-    fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId {
-        self.trait_def_id(tcx)
-    }
-
-    fn probe_and_match_goal_against_assumption(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-        assumption: ty::Clause<'tcx>,
-        then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>,
-    ) -> QueryResult<'tcx> {
-        if let Some(projection_pred) = assumption.as_projection_clause() {
-            if projection_pred.projection_def_id() == goal.predicate.def_id() {
-                let tcx = ecx.tcx();
-                ecx.probe_misc_candidate("assumption").enter(|ecx| {
-                    let assumption_projection_pred =
-                        ecx.instantiate_binder_with_infer(projection_pred);
-                    ecx.eq(
-                        goal.param_env,
-                        goal.predicate.projection_ty,
-                        assumption_projection_pred.projection_ty,
-                    )?;
-                    ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)
-                        .expect("expected goal term to be fully unconstrained");
-
-                    // Add GAT where clauses from the trait's definition
-                    ecx.add_goals(
-                        tcx.predicates_of(goal.predicate.def_id())
-                            .instantiate_own(tcx, goal.predicate.projection_ty.args)
-                            .map(|(pred, _)| goal.with(tcx, pred)),
-                    );
-
-                    then(ecx)
-                })
-            } else {
-                Err(NoSolution)
-            }
-        } else {
-            Err(NoSolution)
-        }
-    }
-
-    fn consider_impl_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
-        impl_def_id: DefId,
-    ) -> Result<Candidate<'tcx>, NoSolution> {
-        let tcx = ecx.tcx();
-
-        let goal_trait_ref = goal.predicate.projection_ty.trait_ref(tcx);
-        let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
-        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::ForLookup };
-        if !drcx.args_may_unify(goal_trait_ref.args, impl_trait_ref.skip_binder().args) {
-            return Err(NoSolution);
-        }
-
-        ecx.probe_trait_candidate(CandidateSource::Impl(impl_def_id)).enter(|ecx| {
-            let impl_args = ecx.fresh_args_for_item(impl_def_id);
-            let impl_trait_ref = impl_trait_ref.instantiate(tcx, impl_args);
-
-            ecx.eq(goal.param_env, goal_trait_ref, impl_trait_ref)?;
-
-            let where_clause_bounds = tcx
-                .predicates_of(impl_def_id)
-                .instantiate(tcx, impl_args)
-                .predicates
-                .into_iter()
-                .map(|pred| goal.with(tcx, pred));
-            ecx.add_goals(where_clause_bounds);
-
-            // Add GAT where clauses from the trait's definition
-            ecx.add_goals(
-                tcx.predicates_of(goal.predicate.def_id())
-                    .instantiate_own(tcx, goal.predicate.projection_ty.args)
-                    .map(|(pred, _)| goal.with(tcx, pred)),
-            );
-
-            // In case the associated item is hidden due to specialization, we have to
-            // return ambiguity this would otherwise be incomplete, resulting in
-            // unsoundness during coherence (#105782).
-            let Some(assoc_def) = fetch_eligible_assoc_item_def(
-                ecx,
-                goal.param_env,
-                goal_trait_ref,
-                goal.predicate.def_id(),
-                impl_def_id,
-            )?
-            else {
-                return ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
-            };
-
-            let error_response = |ecx: &mut EvalCtxt<'_, 'tcx>, reason| {
-                let guar = tcx.sess.span_delayed_bug(tcx.def_span(assoc_def.item.def_id), reason);
-                let error_term = match assoc_def.item.kind {
-                    ty::AssocKind::Const => ty::Const::new_error(
-                        tcx,
-                        guar,
-                        tcx.type_of(goal.predicate.def_id())
-                            .instantiate(tcx, goal.predicate.projection_ty.args),
-                    )
-                    .into(),
-                    ty::AssocKind::Type => Ty::new_error(tcx, guar).into(),
-                    ty::AssocKind::Fn => unreachable!(),
-                };
-                ecx.eq(goal.param_env, goal.predicate.term, error_term)
-                    .expect("expected goal term to be fully unconstrained");
-                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-            };
-
-            if !assoc_def.item.defaultness(tcx).has_value() {
-                return error_response(ecx, "missing value for assoc item in impl");
-            }
-
-            // Getting the right args here is complex, e.g. given:
-            // - a goal `<Vec<u32> as Trait<i32>>::Assoc<u64>`
-            // - the applicable impl `impl<T> Trait<i32> for Vec<T>`
-            // - and the impl which defines `Assoc` being `impl<T, U> Trait<U> for Vec<T>`
-            //
-            // We first rebase the goal args onto the impl, going from `[Vec<u32>, i32, u64]`
-            // to `[u32, u64]`.
-            //
-            // And then map these args to the args of the defining impl of `Assoc`, going
-            // from `[u32, u64]` to `[u32, i32, u64]`.
-            let impl_args_with_gat = goal.predicate.projection_ty.args.rebase_onto(
-                tcx,
-                goal_trait_ref.def_id,
-                impl_args,
-            );
-            let args = ecx.translate_args(
-                goal.param_env,
-                impl_def_id,
-                impl_args_with_gat,
-                assoc_def.defining_node,
-            );
-
-            if !check_args_compatible(tcx, assoc_def.item, args) {
-                return error_response(
-                    ecx,
-                    "associated item has mismatched generic item arguments",
-                );
-            }
-
-            // Finally we construct the actual value of the associated type.
-            let term = match assoc_def.item.kind {
-                ty::AssocKind::Type => tcx.type_of(assoc_def.item.def_id).map_bound(|ty| ty.into()),
-                ty::AssocKind::Const => {
-                    if tcx.features().associated_const_equality {
-                        bug!("associated const projection is not supported yet")
-                    } else {
-                        ty::EarlyBinder::bind(
-                            ty::Const::new_error_with_message(
-                                tcx,
-                                tcx.type_of(assoc_def.item.def_id).instantiate_identity(),
-                                DUMMY_SP,
-                                "associated const projection is not supported yet",
-                            )
-                            .into(),
-                        )
-                    }
-                }
-                ty::AssocKind::Fn => unreachable!("we should never project to a fn"),
-            };
-
-            ecx.eq(goal.param_env, goal.predicate.term, term.instantiate(tcx, args))
-                .expect("expected goal term to be fully unconstrained");
-            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
-    }
-
-    /// Fail to normalize if the predicate contains an error, alternatively, we could normalize to `ty::Error`
-    /// and succeed. Can experiment with this to figure out what results in better error messages.
-    fn consider_error_guaranteed_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        _guar: ErrorGuaranteed,
-    ) -> QueryResult<'tcx> {
-        Err(NoSolution)
-    }
-
-    fn consider_auto_trait_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        ecx.tcx().sess.span_delayed_bug(
-            ecx.tcx().def_span(goal.predicate.def_id()),
-            "associated types not allowed on auto traits",
-        );
-        Err(NoSolution)
-    }
-
-    fn consider_trait_alias_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("trait aliases do not have associated types: {:?}", goal);
-    }
-
-    fn consider_builtin_sized_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`Sized` does not have an associated type: {:?}", goal);
-    }
-
-    fn consider_builtin_copy_clone_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`Copy`/`Clone` does not have an associated type: {:?}", goal);
-    }
-
-    fn consider_builtin_pointer_like_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`PointerLike` does not have an associated type: {:?}", goal);
-    }
-
-    fn consider_builtin_fn_ptr_trait_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`FnPtr` does not have an associated type: {:?}", goal);
-    }
-
-    fn consider_builtin_fn_trait_candidates(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-        goal_kind: ty::ClosureKind,
-    ) -> QueryResult<'tcx> {
-        let tcx = ecx.tcx();
-        let tupled_inputs_and_output =
-            match structural_traits::extract_tupled_inputs_and_output_from_callable(
-                tcx,
-                goal.predicate.self_ty(),
-                goal_kind,
-            )? {
-                Some(tupled_inputs_and_output) => tupled_inputs_and_output,
-                None => {
-                    return ecx
-                        .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
-                }
-            };
-        let output_is_sized_pred = tupled_inputs_and_output.map_bound(|(_, output)| {
-            ty::TraitRef::from_lang_item(tcx, LangItem::Sized, DUMMY_SP, [output])
-        });
-
-        let pred = tupled_inputs_and_output
-            .map_bound(|(inputs, output)| ty::ProjectionPredicate {
-                projection_ty: ty::AliasTy::new(
-                    tcx,
-                    goal.predicate.def_id(),
-                    [goal.predicate.self_ty(), inputs],
-                ),
-                term: output.into(),
-            })
-            .to_predicate(tcx);
-
-        // A built-in `Fn` impl only holds if the output is sized.
-        // (FIXME: technically we only need to check this if the type is a fn ptr...)
-        Self::consider_implied_clause(ecx, goal, pred, [goal.with(tcx, output_is_sized_pred)])
-    }
-
-    fn consider_builtin_tuple_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`Tuple` does not have an associated type: {:?}", goal);
-    }
-
-    fn consider_builtin_pointee_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        let tcx = ecx.tcx();
-        ecx.probe_misc_candidate("builtin pointee").enter(|ecx| {
-            let metadata_ty = match goal.predicate.self_ty().kind() {
-                ty::Bool
-                | ty::Char
-                | ty::Int(..)
-                | ty::Uint(..)
-                | ty::Float(..)
-                | ty::Array(..)
-                | ty::RawPtr(..)
-                | ty::Ref(..)
-                | ty::FnDef(..)
-                | ty::FnPtr(..)
-                | ty::Closure(..)
-                | ty::Infer(ty::IntVar(..) | ty::FloatVar(..))
-                | ty::Coroutine(..)
-                | ty::CoroutineWitness(..)
-                | ty::Never
-                | ty::Foreign(..) => tcx.types.unit,
-
-                ty::Error(e) => Ty::new_error(tcx, *e),
-
-                ty::Str | ty::Slice(_) => tcx.types.usize,
-
-                ty::Dynamic(_, _, _) => {
-                    let dyn_metadata = tcx.require_lang_item(LangItem::DynMetadata, None);
-                    tcx.type_of(dyn_metadata)
-                        .instantiate(tcx, &[ty::GenericArg::from(goal.predicate.self_ty())])
-                }
-
-                ty::Alias(_, _) | ty::Param(_) | ty::Placeholder(..) => {
-                    // FIXME(ptr_metadata): It would also be possible to return a `Ok(Ambig)` with no constraints.
-                    let sized_predicate = ty::TraitRef::from_lang_item(
-                        tcx,
-                        LangItem::Sized,
-                        DUMMY_SP,
-                        [ty::GenericArg::from(goal.predicate.self_ty())],
-                    );
-                    ecx.add_goal(goal.with(tcx, sized_predicate));
-                    tcx.types.unit
-                }
-
-                ty::Adt(def, args) if def.is_struct() => match def.non_enum_variant().tail_opt() {
-                    None => tcx.types.unit,
-                    Some(field_def) => {
-                        let self_ty = field_def.ty(tcx, args);
-                        ecx.add_goal(goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)));
-                        return ecx
-                            .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
-                    }
-                },
-                ty::Adt(_, _) => tcx.types.unit,
-
-                ty::Tuple(elements) => match elements.last() {
-                    None => tcx.types.unit,
-                    Some(&self_ty) => {
-                        ecx.add_goal(goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)));
-                        return ecx
-                            .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
-                    }
-                },
-
-                ty::Infer(
-                    ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_),
-                )
-                | ty::Bound(..) => bug!(
-                    "unexpected self ty `{:?}` when normalizing `<T as Pointee>::Metadata`",
-                    goal.predicate.self_ty()
-                ),
-            };
-
-            ecx.eq(goal.param_env, goal.predicate.term, metadata_ty.into())
-                .expect("expected goal term to be fully unconstrained");
-            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
-    }
-
-    fn consider_builtin_future_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        let self_ty = goal.predicate.self_ty();
-        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
-            return Err(NoSolution);
-        };
-
-        // Coroutines are not futures unless they come from `async` desugaring
-        let tcx = ecx.tcx();
-        if !tcx.coroutine_is_async(def_id) {
-            return Err(NoSolution);
-        }
-
-        let term = args.as_coroutine().return_ty().into();
-
-        Self::consider_implied_clause(
-            ecx,
-            goal,
-            ty::ProjectionPredicate {
-                projection_ty: ty::AliasTy::new(ecx.tcx(), goal.predicate.def_id(), [self_ty]),
-                term,
-            }
-            .to_predicate(tcx),
-            // Technically, we need to check that the future type is Sized,
-            // but that's already proven by the coroutine being WF.
-            [],
-        )
-    }
-
-    fn consider_builtin_iterator_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        let self_ty = goal.predicate.self_ty();
-        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
-            return Err(NoSolution);
-        };
-
-        // Coroutines are not Iterators unless they come from `gen` desugaring
-        let tcx = ecx.tcx();
-        if !tcx.coroutine_is_gen(def_id) {
-            return Err(NoSolution);
-        }
-
-        let term = args.as_coroutine().yield_ty().into();
-
-        Self::consider_implied_clause(
-            ecx,
-            goal,
-            ty::ProjectionPredicate {
-                projection_ty: ty::AliasTy::new(ecx.tcx(), goal.predicate.def_id(), [self_ty]),
-                term,
-            }
-            .to_predicate(tcx),
-            // Technically, we need to check that the iterator type is Sized,
-            // but that's already proven by the generator being WF.
-            [],
-        )
-    }
-
-    fn consider_builtin_coroutine_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        let self_ty = goal.predicate.self_ty();
-        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
-            return Err(NoSolution);
-        };
-
-        // `async`-desugared coroutines do not implement the coroutine trait
-        let tcx = ecx.tcx();
-        if !tcx.is_general_coroutine(def_id) {
-            return Err(NoSolution);
-        }
-
-        let coroutine = args.as_coroutine();
-
-        let name = tcx.associated_item(goal.predicate.def_id()).name;
-        let term = if name == sym::Return {
-            coroutine.return_ty().into()
-        } else if name == sym::Yield {
-            coroutine.yield_ty().into()
-        } else {
-            bug!("unexpected associated item `<{self_ty} as Coroutine>::{name}`")
-        };
-
-        Self::consider_implied_clause(
-            ecx,
-            goal,
-            ty::ProjectionPredicate {
-                projection_ty: ty::AliasTy::new(
-                    ecx.tcx(),
-                    goal.predicate.def_id(),
-                    [self_ty, coroutine.resume_ty()],
-                ),
-                term,
-            }
-            .to_predicate(tcx),
-            // Technically, we need to check that the coroutine type is Sized,
-            // but that's already proven by the coroutine being WF.
-            [],
-        )
-    }
-
-    fn consider_unsize_to_dyn_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`Unsize` does not have an associated type: {:?}", goal)
-    }
-
-    fn consider_structural_builtin_unsize_candidates(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> Vec<(CanonicalResponse<'tcx>, BuiltinImplSource)> {
-        bug!("`Unsize` does not have an associated type: {:?}", goal);
-    }
-
-    fn consider_builtin_discriminant_kind_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        let self_ty = goal.predicate.self_ty();
-        let discriminant_ty = match *self_ty.kind() {
-            ty::Bool
-            | ty::Char
-            | ty::Int(..)
-            | ty::Uint(..)
-            | ty::Float(..)
-            | ty::Array(..)
-            | ty::RawPtr(..)
-            | ty::Ref(..)
-            | ty::FnDef(..)
-            | ty::FnPtr(..)
-            | ty::Closure(..)
-            | ty::Infer(ty::IntVar(..) | ty::FloatVar(..))
-            | ty::Coroutine(..)
-            | ty::CoroutineWitness(..)
-            | ty::Never
-            | ty::Foreign(..)
-            | ty::Adt(_, _)
-            | ty::Str
-            | ty::Slice(_)
-            | ty::Dynamic(_, _, _)
-            | ty::Tuple(_)
-            | ty::Error(_) => self_ty.discriminant_ty(ecx.tcx()),
-
-            // We do not call `Ty::discriminant_ty` on alias, param, or placeholder
-            // types, which return `<self_ty as DiscriminantKind>::Discriminant`
-            // (or ICE in the case of placeholders). Projecting a type to itself
-            // is never really productive.
-            ty::Alias(_, _) | ty::Param(_) | ty::Placeholder(..) => {
-                return Err(NoSolution);
-            }
-
-            ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_))
-            | ty::Bound(..) => bug!(
-                "unexpected self ty `{:?}` when normalizing `<T as DiscriminantKind>::Discriminant`",
-                goal.predicate.self_ty()
-            ),
-        };
-
-        ecx.probe_misc_candidate("builtin discriminant kind").enter(|ecx| {
-            ecx.eq(goal.param_env, goal.predicate.term, discriminant_ty.into())
-                .expect("expected goal term to be fully unconstrained");
-            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
-    }
-
-    fn consider_builtin_destruct_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`Destruct` does not have an associated type: {:?}", goal);
-    }
-
-    fn consider_builtin_transmute_candidate(
-        _ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-    ) -> QueryResult<'tcx> {
-        bug!("`BikeshedIntrinsicFrom` does not have an associated type: {:?}", goal)
-    }
-}
-
-/// This behavior is also implemented in `rustc_ty_utils` and in the old `project` code.
-///
-/// FIXME: We should merge these 3 implementations as it's likely that they otherwise
-/// diverge.
-#[instrument(level = "debug", skip(ecx, param_env), ret)]
-fn fetch_eligible_assoc_item_def<'tcx>(
-    ecx: &EvalCtxt<'_, 'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-    goal_trait_ref: ty::TraitRef<'tcx>,
-    trait_assoc_def_id: DefId,
-    impl_def_id: DefId,
-) -> Result<Option<LeafDef>, NoSolution> {
-    let node_item = specialization_graph::assoc_def(ecx.tcx(), impl_def_id, trait_assoc_def_id)
-        .map_err(|ErrorGuaranteed { .. }| NoSolution)?;
-
-    let eligible = if node_item.is_final() {
-        // Non-specializable items are always projectable.
-        true
-    } else {
-        // Only reveal a specializable default if we're past type-checking
-        // and the obligation is monomorphic, otherwise passes such as
-        // transmute checking and polymorphic MIR optimizations could
-        // get a result which isn't correct for all monomorphizations.
-        if param_env.reveal() == Reveal::All {
-            let poly_trait_ref = ecx.resolve_vars_if_possible(goal_trait_ref);
-            !poly_trait_ref.still_further_specializable()
-        } else {
-            debug!(?node_item.item.def_id, "not eligible due to default");
-            false
-        }
-    };
-
-    if eligible { Ok(Some(node_item)) } else { Ok(None) }
-}
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/opaques.rs b/compiler/rustc_trait_selection/src/solve/project_goals/opaques.rs
deleted file mode 100644
index 1fde129c3a0..00000000000
--- a/compiler/rustc_trait_selection/src/solve/project_goals/opaques.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-//! Computes a normalizes-to (projection) goal for opaque types. This goal
-//! behaves differently depending on the param-env's reveal mode and whether
-//! the opaque is in a defining scope.
-use rustc_middle::traits::query::NoSolution;
-use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
-use rustc_middle::traits::Reveal;
-use rustc_middle::ty;
-use rustc_middle::ty::util::NotUniqueParam;
-
-use crate::solve::{EvalCtxt, SolverMode};
-
-impl<'tcx> EvalCtxt<'_, 'tcx> {
-    pub(super) fn normalize_opaque_type(
-        &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        let tcx = self.tcx();
-        let opaque_ty = goal.predicate.projection_ty;
-        let expected = goal.predicate.term.ty().expect("no such thing as an opaque const");
-
-        match (goal.param_env.reveal(), self.solver_mode()) {
-            (Reveal::UserFacing, SolverMode::Normal) => {
-                let Some(opaque_ty_def_id) = opaque_ty.def_id.as_local() else {
-                    return Err(NoSolution);
-                };
-                // FIXME: at some point we should call queries without defining
-                // new opaque types but having the existing opaque type definitions.
-                // This will require moving this below "Prefer opaques registered already".
-                if !self.can_define_opaque_ty(opaque_ty_def_id) {
-                    return Err(NoSolution);
-                }
-                // FIXME: This may have issues when the args contain aliases...
-                match self.tcx().uses_unique_placeholders_ignoring_regions(opaque_ty.args) {
-                    Err(NotUniqueParam::NotParam(param)) if param.is_non_region_infer() => {
-                        return self.evaluate_added_goals_and_make_canonical_response(
-                            Certainty::AMBIGUOUS,
-                        );
-                    }
-                    Err(_) => {
-                        return Err(NoSolution);
-                    }
-                    Ok(()) => {}
-                }
-                // Prefer opaques registered already.
-                let opaque_type_key =
-                    ty::OpaqueTypeKey { def_id: opaque_ty_def_id, args: opaque_ty.args };
-                // FIXME: This also unifies the previous hidden type with the expected.
-                //
-                // If that fails, we insert `expected` as a new hidden type instead of
-                // eagerly emitting an error.
-                let matches =
-                    self.unify_existing_opaque_tys(goal.param_env, opaque_type_key, expected);
-                if !matches.is_empty() {
-                    if let Some(response) = self.try_merge_responses(&matches) {
-                        return Ok(response);
-                    } else {
-                        return self.flounder(&matches);
-                    }
-                }
-
-                let expected = match self.try_normalize_ty(goal.param_env, expected) {
-                    Some(ty) => {
-                        if ty.is_ty_var() {
-                            return self.evaluate_added_goals_and_make_canonical_response(
-                                Certainty::AMBIGUOUS,
-                            );
-                        } else {
-                            ty
-                        }
-                    }
-                    None => {
-                        return self
-                            .evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
-                    }
-                };
-
-                // Otherwise, define a new opaque type
-                self.insert_hidden_type(opaque_type_key, goal.param_env, expected)?;
-                self.add_item_bounds_for_hidden_type(
-                    opaque_ty.def_id,
-                    opaque_ty.args,
-                    goal.param_env,
-                    expected,
-                );
-                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-            }
-            (Reveal::UserFacing, SolverMode::Coherence) => {
-                // An impossible opaque type bound is the only way this goal will fail
-                // e.g. assigning `impl Copy := NotCopy`
-                self.add_item_bounds_for_hidden_type(
-                    opaque_ty.def_id,
-                    opaque_ty.args,
-                    goal.param_env,
-                    expected,
-                );
-                self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
-            }
-            (Reveal::All, _) => {
-                // FIXME: Add an assertion that opaque type storage is empty.
-                let actual = tcx.type_of(opaque_ty.def_id).instantiate(tcx, opaque_ty.args);
-                self.eq(goal.param_env, expected, actual)?;
-                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-            }
-        }
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/weak_types.rs b/compiler/rustc_trait_selection/src/solve/project_goals/weak_types.rs
deleted file mode 100644
index 54de32cf618..00000000000
--- a/compiler/rustc_trait_selection/src/solve/project_goals/weak_types.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-//! Computes a normalizes-to (projection) goal for inherent associated types,
-//! `#![feature(lazy_type_alias)]` and `#![feature(type_alias_impl_trait)]`.
-//!
-//! Since a weak alias is not ambiguous, this just computes the `type_of` of
-//! the alias and registers the where-clauses of the type alias.
-use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
-use rustc_middle::ty;
-
-use super::EvalCtxt;
-
-impl<'tcx> EvalCtxt<'_, 'tcx> {
-    pub(super) fn normalize_weak_type(
-        &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        let tcx = self.tcx();
-        let weak_ty = goal.predicate.projection_ty;
-        let expected = goal.predicate.term.ty().expect("no such thing as a const alias");
-
-        let actual = tcx.type_of(weak_ty.def_id).instantiate(tcx, weak_ty.args);
-        self.eq(goal.param_env, expected, actual)?;
-
-        // Check where clauses
-        self.add_goals(
-            tcx.predicates_of(weak_ty.def_id)
-                .instantiate(tcx, weak_ty.args)
-                .predicates
-                .into_iter()
-                .map(|pred| goal.with(tcx, pred)),
-        );
-
-        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/traits/structural_normalize.rs b/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
index 9d6be768901..32de8feda81 100644
--- a/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
@@ -25,8 +25,7 @@ impl<'tcx> StructurallyNormalizeExt<'tcx> for At<'_, 'tcx> {
             // FIXME(-Ztrait-solver=next): correctly handle
             // overflow here.
             for _ in 0..256 {
-                let ty::Alias(ty::Projection | ty::Inherent | ty::Weak, projection_ty) = *ty.kind()
-                else {
+                let ty::Alias(ty::Projection | ty::Inherent | ty::Weak, alias) = *ty.kind() else {
                     break;
                 };
 
@@ -38,10 +37,7 @@ impl<'tcx> StructurallyNormalizeExt<'tcx> for At<'_, 'tcx> {
                     self.infcx.tcx,
                     self.cause.clone(),
                     self.param_env,
-                    ty::Binder::dummy(ty::ProjectionPredicate {
-                        projection_ty,
-                        term: new_infer_ty.into(),
-                    }),
+                    ty::NormalizesTo { alias, term: new_infer_ty.into() },
                 );
                 if self.infcx.predicate_may_hold(&obligation) {
                     fulfill_cx.register_predicate_obligation(self.infcx, obligation);
diff --git a/compiler/rustc_type_ir/src/predicate_kind.rs b/compiler/rustc_type_ir/src/predicate_kind.rs
index adeb3f57497..b567fa8e2f6 100644
--- a/compiler/rustc_type_ir/src/predicate_kind.rs
+++ b/compiler/rustc_type_ir/src/predicate_kind.rs
@@ -120,7 +120,7 @@ where
 }
 
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""), Hash(bound = ""))]
+#[derivative(Clone(bound = ""), Hash(bound = ""), PartialEq(bound = ""), Eq(bound = ""))]
 #[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable, HashStable_NoContext))]
 pub enum PredicateKind<I: Interner> {
     /// Prove a clause
@@ -169,7 +169,6 @@ pub enum PredicateKind<I: Interner> {
     AliasRelate(I::Term, I::Term, AliasRelationDirection),
 }
 
-/// FIXME: This impl sh
 impl<I: Interner> Copy for PredicateKind<I>
 where
     I::DefId: Copy,
@@ -183,24 +182,6 @@ where
 {
 }
 
-impl<I: Interner> PartialEq for PredicateKind<I> {
-    fn eq(&self, other: &Self) -> bool {
-        match (self, other) {
-            (Self::Clause(l0), Self::Clause(r0)) => l0 == r0,
-            (Self::ObjectSafe(l0), Self::ObjectSafe(r0)) => l0 == r0,
-            (Self::Subtype(l0), Self::Subtype(r0)) => l0 == r0,
-            (Self::Coerce(l0), Self::Coerce(r0)) => l0 == r0,
-            (Self::ConstEquate(l0, l1), Self::ConstEquate(r0, r1)) => l0 == r0 && l1 == r1,
-            (Self::AliasRelate(l0, l1, l2), Self::AliasRelate(r0, r1, r2)) => {
-                l0 == r0 && l1 == r1 && l2 == r2
-            }
-            _ => core::mem::discriminant(self) == core::mem::discriminant(other),
-        }
-    }
-}
-
-impl<I: Interner> Eq for PredicateKind<I> {}
-
 impl<I: Interner> TypeFoldable<I> for PredicateKind<I>
 where
     I::DefId: TypeFoldable<I>,
diff --git a/tests/ui/traits/new-solver/alias-bound-unsound.rs b/tests/ui/traits/new-solver/alias-bound-unsound.rs
index 825e874d71b..907a3010355 100644
--- a/tests/ui/traits/new-solver/alias-bound-unsound.rs
+++ b/tests/ui/traits/new-solver/alias-bound-unsound.rs
@@ -23,7 +23,7 @@ fn main() {
     let x = String::from("hello, world");
     drop(<() as Foo>::copy_me(&x));
     //~^ ERROR overflow evaluating the requirement `<() as Foo>::Item: Sized`
-    //~| ERROR overflow evaluating the requirement `<() as Foo>::Item == _`
+    //~| ERROR overflow evaluating the requirement `<() as Foo>::Item normalizes-to _`
     //~| ERROR overflow evaluating the requirement `<() as Foo>::Item well-formed`
     //~| ERROR overflow evaluating the requirement `String <: <() as Foo>::Item`
     //~| ERROR overflow evaluating the requirement `&<() as Foo>::Item well-formed`
diff --git a/tests/ui/traits/new-solver/alias-bound-unsound.stderr b/tests/ui/traits/new-solver/alias-bound-unsound.stderr
index ca4b5c90ff2..29d4d983c03 100644
--- a/tests/ui/traits/new-solver/alias-bound-unsound.stderr
+++ b/tests/ui/traits/new-solver/alias-bound-unsound.stderr
@@ -19,7 +19,7 @@ LL |     drop(<() as Foo>::copy_me(&x));
    |
    = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`alias_bound_unsound`)
 
-error[E0275]: overflow evaluating the requirement `<() as Foo>::Item == _`
+error[E0275]: overflow evaluating the requirement `<() as Foo>::Item normalizes-to _`
   --> $DIR/alias-bound-unsound.rs:24:10
    |
 LL |     drop(<() as Foo>::copy_me(&x));
diff --git a/tests/ui/traits/new-solver/generalize/generalize-proj-new-universe-index-2.rs b/tests/ui/traits/new-solver/generalize/generalize-proj-new-universe-index-2.rs
index 94d645a9859..7fe242f4714 100644
--- a/tests/ui/traits/new-solver/generalize/generalize-proj-new-universe-index-2.rs
+++ b/tests/ui/traits/new-solver/generalize/generalize-proj-new-universe-index-2.rs
@@ -1,5 +1,7 @@
 // compile-flags: -Ztrait-solver=next
 // known-bug: trait-system-refactor-initiative#60
+// dont-check-failure-status
+// dont-check-compiler-stderr
 
 // Generalizing a projection containing an inference variable
 // which cannot be named by the `root_vid` can result in ambiguity.
diff --git a/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.next.stderr b/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.next.stderr
index 34c2f0438c7..ad8b24a39c7 100644
--- a/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.next.stderr
+++ b/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.next.stderr
@@ -1,9 +1,11 @@
-error[E0271]: type mismatch resolving `<<T as Id<_>>::Id as Unnormalizable>::Assoc == _`
-  --> $DIR/occurs-check-nested-alias.rs:35:9
+error[E0275]: overflow evaluating the requirement `<<T as Id<_>>::Id as Unnormalizable>::Assoc == _`
+  --> $DIR/occurs-check-nested-alias.rs:36:9
    |
 LL |     x = y;
-   |         ^ types differ
+   |         ^
+   |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`occurs_check_nested_alias`)
 
 error: aborting due to 1 previous error
 
-For more information about this error, try `rustc --explain E0271`.
+For more information about this error, try `rustc --explain E0275`.
diff --git a/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.rs b/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.rs
index a2113b2a8b3..02ac091c0a8 100644
--- a/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.rs
+++ b/tests/ui/traits/new-solver/generalize/occurs-check-nested-alias.rs
@@ -1,7 +1,8 @@
 // revisions: old next
 //[old] check-pass
 
-// Need to emit an alias-relate instead of a `Projection` goal here.
+// Currently always fails to generalize the outer alias, even if it
+// is treated as rigid by `alias-relate`.
 //[next] compile-flags: -Ztrait-solver=next
 //[next] known-bug: trait-system-refactor-initiative#8
 #![crate_type = "lib"]
diff --git a/tests/ui/traits/new-solver/specialization-transmute.stderr b/tests/ui/traits/new-solver/specialization-transmute.stderr
index 18965a465b3..eaf32a475ac 100644
--- a/tests/ui/traits/new-solver/specialization-transmute.stderr
+++ b/tests/ui/traits/new-solver/specialization-transmute.stderr
@@ -8,13 +8,11 @@ LL | #![feature(specialization)]
    = help: consider using `min_specialization` instead, which is more stable and complete
    = note: `#[warn(incomplete_features)]` on by default
 
-error[E0284]: type annotations needed
+error[E0284]: type annotations needed: cannot satisfy `<T as Default>::Id normalizes-to _`
   --> $DIR/specialization-transmute.rs:15:23
    |
 LL |     fn intu(&self) -> &Self::Id {
-   |                       ^^^^^^^^^ cannot infer type
-   |
-   = note: cannot satisfy `<T as Default>::Id == _`
+   |                       ^^^^^^^^^ cannot satisfy `<T as Default>::Id normalizes-to _`
 
 error[E0282]: type annotations needed
   --> $DIR/specialization-transmute.rs:13:23
-- 
cgit 1.4.1-3-g733a5


From 1490c58076d019eacf1d0db3dbe93299ecc68a15 Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Fri, 8 Dec 2023 01:34:40 +0100
Subject: add regression tests

---
 .../projection/param-env-trait-candidate-1.rs      | 14 +++++++++++
 .../projection/param-env-trait-candidate-2.rs      | 29 ++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 tests/ui/traits/new-solver/projection/param-env-trait-candidate-1.rs
 create mode 100644 tests/ui/traits/new-solver/projection/param-env-trait-candidate-2.rs

diff --git a/tests/ui/traits/new-solver/projection/param-env-trait-candidate-1.rs b/tests/ui/traits/new-solver/projection/param-env-trait-candidate-1.rs
new file mode 100644
index 00000000000..e36d574efe2
--- /dev/null
+++ b/tests/ui/traits/new-solver/projection/param-env-trait-candidate-1.rs
@@ -0,0 +1,14 @@
+// check-pass
+// compile-flags: -Ztrait-solver=next
+
+// See https://github.com/rust-lang/trait-system-refactor-initiative/issues/1
+// a minimization of a pattern in core.
+fn next<T: Iterator<Item = U>, U>(t: &mut T) -> Option<U> {
+    t.next()
+}
+
+fn foo<T: Iterator>(t: &mut T) {
+    let _: Option<T::Item> = next(t);
+}
+
+fn main() {}
diff --git a/tests/ui/traits/new-solver/projection/param-env-trait-candidate-2.rs b/tests/ui/traits/new-solver/projection/param-env-trait-candidate-2.rs
new file mode 100644
index 00000000000..c8050997a1d
--- /dev/null
+++ b/tests/ui/traits/new-solver/projection/param-env-trait-candidate-2.rs
@@ -0,0 +1,29 @@
+// check-pass
+// compile-flags: -Ztrait-solver=next
+
+// See https://github.com/rust-lang/trait-system-refactor-initiative/issues/1,
+// a minimization of a pattern in core.
+
+trait Iterator {
+    type Item;
+}
+
+struct Flatten<I>(I);
+
+impl<I, U> Iterator for Flatten<I>
+where
+    I: Iterator<Item = U>,
+{
+    type Item = U;
+}
+
+fn needs_iterator<I: Iterator>() {}
+
+fn environment<J>()
+where
+    J: Iterator,
+{
+    needs_iterator::<Flatten<J>>();
+}
+
+fn main() {}
-- 
cgit 1.4.1-3-g733a5


From 4c9e842a09bdde35bf76bc4690f665dfbac3bfd1 Mon Sep 17 00:00:00 2001
From: "Celina G. Val" <celinval@amazon.com>
Date: Wed, 6 Dec 2023 13:39:55 -0800
Subject: Add instance evaluation and methods to read alloc

The instance evaluation is needed to handle intrinsics such as
`type_id` and `type_name`.

Since we now use Allocation to represent all evaluated constants,
provide a few methods to help process the data inside an allocation.
---
 Cargo.lock                                         |   1 +
 compiler/rustc_smir/Cargo.toml                     |   1 +
 compiler/rustc_smir/src/rustc_smir/alloc.rs        |  37 +++++---
 compiler/rustc_smir/src/rustc_smir/context.rs      |  30 ++++++-
 .../rustc_smir/src/rustc_smir/convert/error.rs     |  22 +++++
 compiler/rustc_smir/src/rustc_smir/convert/mod.rs  |  12 +++
 compiler/stable_mir/src/compiler_interface.rs      |   7 ++
 compiler/stable_mir/src/error.rs                   |  10 ++-
 compiler/stable_mir/src/lib.rs                     |   1 +
 compiler/stable_mir/src/mir/alloc.rs               |  36 +++++++-
 compiler/stable_mir/src/mir/body.rs                |   2 +-
 compiler/stable_mir/src/mir/mono.rs                |   8 ++
 compiler/stable_mir/src/target.rs                  |  50 +++++++++++
 compiler/stable_mir/src/ty.rs                      | 100 ++++++++++++++++++++-
 14 files changed, 295 insertions(+), 22 deletions(-)
 create mode 100644 compiler/rustc_smir/src/rustc_smir/convert/error.rs
 create mode 100644 compiler/stable_mir/src/target.rs

diff --git a/Cargo.lock b/Cargo.lock
index 7f627e2ce6e..e56f05a5b19 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4492,6 +4492,7 @@ dependencies = [
 name = "rustc_smir"
 version = "0.0.0"
 dependencies = [
+ "rustc_abi",
  "rustc_data_structures",
  "rustc_hir",
  "rustc_middle",
diff --git a/compiler/rustc_smir/Cargo.toml b/compiler/rustc_smir/Cargo.toml
index 836ea046ffe..c9e23efcb10 100644
--- a/compiler/rustc_smir/Cargo.toml
+++ b/compiler/rustc_smir/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
+rustc_abi = { path = "../rustc_abi" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_middle = { path = "../rustc_middle" }
diff --git a/compiler/rustc_smir/src/rustc_smir/alloc.rs b/compiler/rustc_smir/src/rustc_smir/alloc.rs
index 850a52ce275..48cb164c308 100644
--- a/compiler/rustc_smir/src/rustc_smir/alloc.rs
+++ b/compiler/rustc_smir/src/rustc_smir/alloc.rs
@@ -2,6 +2,7 @@ use rustc_middle::mir::{
     interpret::{alloc_range, AllocRange, Pointer},
     ConstValue,
 };
+use stable_mir::Error;
 
 use crate::rustc_smir::{Stable, Tables};
 use stable_mir::mir::Mutability;
@@ -26,23 +27,35 @@ pub fn new_allocation<'tcx>(
     const_value: ConstValue<'tcx>,
     tables: &mut Tables<'tcx>,
 ) -> Allocation {
-    match const_value {
+    try_new_allocation(ty, const_value, tables).unwrap()
+}
+
+#[allow(rustc::usage_of_qualified_ty)]
+pub fn try_new_allocation<'tcx>(
+    ty: rustc_middle::ty::Ty<'tcx>,
+    const_value: ConstValue<'tcx>,
+    tables: &mut Tables<'tcx>,
+) -> Result<Allocation, Error> {
+    Ok(match const_value {
         ConstValue::Scalar(scalar) => {
             let size = scalar.size();
             let align = tables
                 .tcx
                 .layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty))
-                .unwrap()
+                .map_err(|e| e.stable(tables))?
                 .align;
             let mut allocation = rustc_middle::mir::interpret::Allocation::uninit(size, align.abi);
             allocation
                 .write_scalar(&tables.tcx, alloc_range(rustc_target::abi::Size::ZERO, size), scalar)
-                .unwrap();
+                .map_err(|e| e.stable(tables))?;
             allocation.stable(tables)
         }
         ConstValue::ZeroSized => {
-            let align =
-                tables.tcx.layout_of(rustc_middle::ty::ParamEnv::empty().and(ty)).unwrap().align;
+            let align = tables
+                .tcx
+                .layout_of(rustc_middle::ty::ParamEnv::empty().and(ty))
+                .map_err(|e| e.stable(tables))?
+                .align;
             new_empty_allocation(align.abi)
         }
         ConstValue::Slice { data, meta } => {
@@ -51,8 +64,10 @@ pub fn new_allocation<'tcx>(
             let scalar_ptr = rustc_middle::mir::interpret::Scalar::from_pointer(ptr, &tables.tcx);
             let scalar_meta =
                 rustc_middle::mir::interpret::Scalar::from_target_usize(meta, &tables.tcx);
-            let layout =
-                tables.tcx.layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty)).unwrap();
+            let layout = tables
+                .tcx
+                .layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty))
+                .map_err(|e| e.stable(tables))?;
             let mut allocation =
                 rustc_middle::mir::interpret::Allocation::uninit(layout.size, layout.align.abi);
             allocation
@@ -61,14 +76,14 @@ pub fn new_allocation<'tcx>(
                     alloc_range(rustc_target::abi::Size::ZERO, tables.tcx.data_layout.pointer_size),
                     scalar_ptr,
                 )
-                .unwrap();
+                .map_err(|e| e.stable(tables))?;
             allocation
                 .write_scalar(
                     &tables.tcx,
                     alloc_range(tables.tcx.data_layout.pointer_size, scalar_meta.size()),
                     scalar_meta,
                 )
-                .unwrap();
+                .map_err(|e| e.stable(tables))?;
             allocation.stable(tables)
         }
         ConstValue::Indirect { alloc_id, offset } => {
@@ -76,11 +91,11 @@ pub fn new_allocation<'tcx>(
             let ty_size = tables
                 .tcx
                 .layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty))
-                .unwrap()
+                .map_err(|e| e.stable(tables))?
                 .size;
             allocation_filter(&alloc.0, alloc_range(offset, ty_size), tables)
         }
-    }
+    })
 }
 
 /// Creates an `Allocation` only from information within the `AllocRange`.
diff --git a/compiler/rustc_smir/src/rustc_smir/context.rs b/compiler/rustc_smir/src/rustc_smir/context.rs
index b10dfe85914..341c69e9327 100644
--- a/compiler/rustc_smir/src/rustc_smir/context.rs
+++ b/compiler/rustc_smir/src/rustc_smir/context.rs
@@ -11,18 +11,29 @@ use stable_mir::compiler_interface::Context;
 use stable_mir::mir::alloc::GlobalAlloc;
 use stable_mir::mir::mono::{InstanceDef, StaticDef};
 use stable_mir::mir::Body;
+use stable_mir::target::{MachineInfo, MachineSize};
 use stable_mir::ty::{
     AdtDef, AdtKind, Allocation, ClosureDef, ClosureKind, Const, FieldDef, FnDef, GenericArgs,
-    LineInfo, PolyFnSig, RigidTy, Span, TyKind, VariantDef,
+    LineInfo, PolyFnSig, RigidTy, Span, Ty, TyKind, VariantDef,
 };
 use stable_mir::{self, Crate, CrateItem, DefId, Error, Filename, ItemKind, Symbol};
 use std::cell::RefCell;
 
 use crate::rustc_internal::{internal, RustcInternal};
 use crate::rustc_smir::builder::BodyBuilder;
-use crate::rustc_smir::{new_item_kind, smir_crate, Stable, Tables};
+use crate::rustc_smir::{alloc, new_item_kind, smir_crate, Stable, Tables};
 
 impl<'tcx> Context for TablesWrapper<'tcx> {
+    fn target_info(&self) -> MachineInfo {
+        let mut tables = self.0.borrow_mut();
+        MachineInfo {
+            endian: tables.tcx.data_layout.endian.stable(&mut *tables),
+            pointer_width: MachineSize::from_bits(
+                tables.tcx.data_layout.pointer_size.bits().try_into().unwrap(),
+            ),
+        }
+    }
+
     fn entry_fn(&self) -> Option<stable_mir::CrateItem> {
         let mut tables = self.0.borrow_mut();
         let tcx = tables.tcx;
@@ -382,6 +393,21 @@ impl<'tcx> Context for TablesWrapper<'tcx> {
         Instance::resolve_closure(tables.tcx, def_id, args_ref, closure_kind).stable(&mut *tables)
     }
 
+    fn eval_instance(&self, def: InstanceDef, const_ty: Ty) -> Result<Allocation, Error> {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        let result = tables.tcx.const_eval_instance(
+            ParamEnv::reveal_all(),
+            instance,
+            Some(tables.tcx.def_span(instance.def_id())),
+        );
+        result
+            .map(|const_val| {
+                alloc::try_new_allocation(const_ty.internal(&mut *tables), const_val, &mut *tables)
+            })
+            .map_err(|e| e.stable(&mut *tables))?
+    }
+
     fn eval_static_initializer(&self, def: StaticDef) -> Result<Allocation, Error> {
         let mut tables = self.0.borrow_mut();
         let def_id = def.0.internal(&mut *tables);
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/error.rs b/compiler/rustc_smir/src/rustc_smir/convert/error.rs
new file mode 100644
index 00000000000..6c582b799f8
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/convert/error.rs
@@ -0,0 +1,22 @@
+//! Handle the conversion of different internal errors into a stable version.
+//!
+//! Currently we encode everything as [stable_mir::Error], which is represented as a string.
+use crate::rustc_smir::{Stable, Tables};
+use rustc_middle::mir::interpret::AllocError;
+use rustc_middle::ty::layout::LayoutError;
+
+impl<'tcx> Stable<'tcx> for LayoutError<'tcx> {
+    type T = stable_mir::Error;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::Error::new(format!("{self:?}"))
+    }
+}
+
+impl<'tcx> Stable<'tcx> for AllocError {
+    type T = stable_mir::Error;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::Error::new(format!("{self:?}"))
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mod.rs b/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
index 9c0b2b29bca..a83ff0663b9 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
@@ -5,6 +5,7 @@ use stable_mir::ty::{IndexedVal, VariantIdx};
 
 use crate::rustc_smir::{Stable, Tables};
 
+mod error;
 mod mir;
 mod ty;
 
@@ -75,3 +76,14 @@ impl<'tcx> Stable<'tcx> for rustc_span::Span {
         tables.create_span(*self)
     }
 }
+
+impl<'tcx> Stable<'tcx> for rustc_abi::Endian {
+    type T = stable_mir::target::Endian;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            rustc_abi::Endian::Little => stable_mir::target::Endian::Little,
+            rustc_abi::Endian::Big => stable_mir::target::Endian::Big,
+        }
+    }
+}
diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs
index 7916d04250d..928f320c2fa 100644
--- a/compiler/stable_mir/src/compiler_interface.rs
+++ b/compiler/stable_mir/src/compiler_interface.rs
@@ -8,6 +8,7 @@ use std::cell::Cell;
 use crate::mir::alloc::{AllocId, GlobalAlloc};
 use crate::mir::mono::{Instance, InstanceDef, StaticDef};
 use crate::mir::Body;
+use crate::target::MachineInfo;
 use crate::ty::{
     AdtDef, AdtKind, Allocation, ClosureDef, ClosureKind, Const, FieldDef, FnDef, GenericArgs,
     GenericPredicates, Generics, ImplDef, ImplTrait, LineInfo, PolyFnSig, RigidTy, Span, TraitDecl,
@@ -150,6 +151,9 @@ pub trait Context {
     /// Evaluate a static's initializer.
     fn eval_static_initializer(&self, def: StaticDef) -> Result<Allocation, Error>;
 
+    /// Try to evaluate an instance into a constant.
+    fn eval_instance(&self, def: InstanceDef, const_ty: Ty) -> Result<Allocation, Error>;
+
     /// Retrieve global allocation for the given allocation ID.
     fn global_alloc(&self, id: AllocId) -> GlobalAlloc;
 
@@ -157,6 +161,9 @@ pub trait Context {
     fn vtable_allocation(&self, global_alloc: &GlobalAlloc) -> Option<AllocId>;
     fn krate(&self, def_id: DefId) -> Crate;
     fn instance_name(&self, def: InstanceDef, trimmed: bool) -> Symbol;
+
+    /// Return the number of bytes for a pointer size.
+    fn target_info(&self) -> MachineInfo;
 }
 
 // A thread local variable that stores a pointer to the tables mapping between TyCtxt
diff --git a/compiler/stable_mir/src/error.rs b/compiler/stable_mir/src/error.rs
index bb5e1a34180..c6da3ae41d3 100644
--- a/compiler/stable_mir/src/error.rs
+++ b/compiler/stable_mir/src/error.rs
@@ -6,11 +6,11 @@
 
 use std::convert::From;
 use std::fmt::{Debug, Display, Formatter};
-use std::{error, fmt};
+use std::{error, fmt, io};
 
 macro_rules! error {
      ($fmt: literal $(,)?) => { Error(format!($fmt)) };
-     ($fmt: literal, $($arg:tt)*) => { Error(format!($fmt, $($arg:tt)*)) };
+     ($fmt: literal, $($arg:tt)*) => { Error(format!($fmt, $($arg)*)) };
  }
 
 /// An error type used to represent an error that has already been reported by the compiler.
@@ -79,3 +79,9 @@ where
 
 impl error::Error for Error {}
 impl<T> error::Error for CompilerError<T> where T: Display + Debug {}
+
+impl From<io::Error> for Error {
+    fn from(value: io::Error) -> Self {
+        Error(value.to_string())
+    }
+}
diff --git a/compiler/stable_mir/src/lib.rs b/compiler/stable_mir/src/lib.rs
index 1e7495009d8..8c66bfb2e98 100644
--- a/compiler/stable_mir/src/lib.rs
+++ b/compiler/stable_mir/src/lib.rs
@@ -39,6 +39,7 @@ pub mod compiler_interface;
 #[macro_use]
 pub mod error;
 pub mod mir;
+pub mod target;
 pub mod ty;
 pub mod visitor;
 
diff --git a/compiler/stable_mir/src/mir/alloc.rs b/compiler/stable_mir/src/mir/alloc.rs
index af951bcef8c..c780042ff26 100644
--- a/compiler/stable_mir/src/mir/alloc.rs
+++ b/compiler/stable_mir/src/mir/alloc.rs
@@ -1,7 +1,9 @@
 //! This module provides methods to retrieve allocation information, such as static variables.
 use crate::mir::mono::{Instance, StaticDef};
+use crate::target::{Endian, MachineInfo};
 use crate::ty::{Allocation, Binder, ExistentialTraitRef, IndexedVal, Ty};
-use crate::with;
+use crate::{with, Error};
+use std::io::Read;
 
 /// An allocation in the SMIR global memory can be either a function pointer,
 /// a static, or a "real" allocation with some data in it.
@@ -38,7 +40,7 @@ impl GlobalAlloc {
 }
 
 /// A unique identification number for each provenance
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
 pub struct AllocId(usize);
 
 impl IndexedVal for AllocId {
@@ -49,3 +51,33 @@ impl IndexedVal for AllocId {
         self.0
     }
 }
+
+/// Utility function used to read an allocation data into a unassigned integer.
+pub(crate) fn read_target_uint(mut bytes: &[u8]) -> Result<u128, Error> {
+    let mut buf = [0u8; std::mem::size_of::<u128>()];
+    match MachineInfo::target_endianess() {
+        Endian::Little => {
+            bytes.read(&mut buf)?;
+            Ok(u128::from_le_bytes(buf))
+        }
+        Endian::Big => {
+            bytes.read(&mut buf[16 - bytes.len()..])?;
+            Ok(u128::from_be_bytes(buf))
+        }
+    }
+}
+
+/// Utility function used to read an allocation data into an assigned integer.
+pub(crate) fn read_target_int(mut bytes: &[u8]) -> Result<i128, Error> {
+    let mut buf = [0u8; std::mem::size_of::<i128>()];
+    match MachineInfo::target_endianess() {
+        Endian::Little => {
+            bytes.read(&mut buf)?;
+            Ok(i128::from_le_bytes(buf))
+        }
+        Endian::Big => {
+            bytes.read(&mut buf[16 - bytes.len()..])?;
+            Ok(i128::from_be_bytes(buf))
+        }
+    }
+}
diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index 90bd7aa7d18..d03ded20e9b 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -832,7 +832,7 @@ pub enum MutBorrowKind {
     ClosureCapture,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum Mutability {
     Not,
     Mut,
diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs
index 5c27f9281de..acf268e56b3 100644
--- a/compiler/stable_mir/src/mir/mono.rs
+++ b/compiler/stable_mir/src/mir/mono.rs
@@ -132,6 +132,14 @@ impl Instance {
     pub fn is_empty_shim(&self) -> bool {
         self.kind == InstanceKind::Shim && with(|cx| cx.is_empty_drop_shim(self.def))
     }
+
+    /// Try to constant evaluate the instance into a constant with the given type.
+    ///
+    /// This can be used to retrieve a constant that represents an intrinsic return such as
+    /// `type_id`.
+    pub fn try_const_eval(&self, const_ty: Ty) -> Result<Allocation, Error> {
+        with(|cx| cx.eval_instance(self.def, const_ty))
+    }
 }
 
 impl Debug for Instance {
diff --git a/compiler/stable_mir/src/target.rs b/compiler/stable_mir/src/target.rs
new file mode 100644
index 00000000000..bed1dbc4c00
--- /dev/null
+++ b/compiler/stable_mir/src/target.rs
@@ -0,0 +1,50 @@
+//! Provide information about the machine that this is being compiled into.
+
+use crate::compiler_interface::with;
+
+/// The properties of the target machine being compiled into.
+#[derive(Clone, PartialEq, Eq)]
+pub struct MachineInfo {
+    pub endian: Endian,
+    pub pointer_width: MachineSize,
+}
+
+impl MachineInfo {
+    pub fn target() -> MachineInfo {
+        with(|cx| cx.target_info().clone())
+    }
+
+    pub fn target_endianess() -> Endian {
+        with(|cx| cx.target_info().endian)
+    }
+
+    pub fn target_pointer_width() -> MachineSize {
+        with(|cx| cx.target_info().pointer_width)
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+pub enum Endian {
+    Little,
+    Big,
+}
+
+/// Represent the size of a component.
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct MachineSize {
+    num_bits: usize,
+}
+
+impl MachineSize {
+    pub fn bytes(self) -> usize {
+        self.num_bits / 8
+    }
+
+    pub fn bits(self) -> usize {
+        self.num_bits
+    }
+
+    pub fn from_bits(num_bits: usize) -> MachineSize {
+        MachineSize { num_bits }
+    }
+}
diff --git a/compiler/stable_mir/src/ty.rs b/compiler/stable_mir/src/ty.rs
index c922264f8a3..4caac575c1e 100644
--- a/compiler/stable_mir/src/ty.rs
+++ b/compiler/stable_mir/src/ty.rs
@@ -4,9 +4,11 @@ use super::{
     with, DefId, Error, Symbol,
 };
 use crate::crate_def::CrateDef;
-use crate::mir::alloc::AllocId;
+use crate::mir::alloc::{read_target_int, read_target_uint, AllocId};
+use crate::target::MachineInfo;
 use crate::{Filename, Opaque};
 use std::fmt::{self, Debug, Display, Formatter};
+use std::ops::Range;
 
 #[derive(Copy, Clone, Eq, PartialEq, Hash)]
 pub struct Ty(pub usize);
@@ -366,6 +368,19 @@ pub enum IntTy {
     I128,
 }
 
+impl IntTy {
+    pub fn num_bytes(self) -> usize {
+        match self {
+            IntTy::Isize => crate::target::MachineInfo::target_pointer_width().bytes().into(),
+            IntTy::I8 => 1,
+            IntTy::I16 => 2,
+            IntTy::I32 => 4,
+            IntTy::I64 => 8,
+            IntTy::I128 => 16,
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum UintTy {
     Usize,
@@ -376,6 +391,19 @@ pub enum UintTy {
     U128,
 }
 
+impl UintTy {
+    pub fn num_bytes(self) -> usize {
+        match self {
+            UintTy::Usize => crate::target::MachineInfo::target_pointer_width().bytes().into(),
+            UintTy::U8 => 1,
+            UintTy::U16 => 2,
+            UintTy::U32 => 4,
+            UintTy::U64 => 8,
+            UintTy::U128 => 16,
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum FloatTy {
     F32,
@@ -821,21 +849,21 @@ pub struct BoundTy {
 pub type Bytes = Vec<Option<u8>>;
 pub type Size = usize;
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
 pub struct Prov(pub AllocId);
 pub type Align = u64;
 pub type Promoted = u32;
 pub type InitMaskMaterialized = Vec<u64>;
 
 /// Stores the provenance information of pointers stored in memory.
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
 pub struct ProvenanceMap {
     /// Provenance in this map applies from the given offset for an entire pointer-size worth of
     /// bytes. Two entries in this map are always at least a pointer size apart.
     pub ptrs: Vec<(Size, Prov)>,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
 pub struct Allocation {
     pub bytes: Bytes,
     pub provenance: ProvenanceMap,
@@ -843,6 +871,70 @@ pub struct Allocation {
     pub mutability: Mutability,
 }
 
+impl Allocation {
+    /// Get a vector of bytes for an Allocation that has been fully initialized
+    pub fn raw_bytes(&self) -> Result<Vec<u8>, Error> {
+        self.bytes
+            .iter()
+            .copied()
+            .collect::<Option<Vec<_>>>()
+            .ok_or_else(|| error!("Found uninitialized bytes: `{:?}`", self.bytes))
+    }
+
+    /// Read a uint value from the specified range.
+    pub fn read_partial_uint(&self, range: Range<usize>) -> Result<u128, Error> {
+        if range.end - range.start > 16 {
+            return Err(error!("Allocation is bigger than largest integer"));
+        }
+        if range.end > self.bytes.len() {
+            return Err(error!(
+                "Range is out of bounds. Allocation length is `{}`, but requested range `{:?}`",
+                self.bytes.len(),
+                range
+            ));
+        }
+        let raw = self.bytes[range]
+            .iter()
+            .copied()
+            .collect::<Option<Vec<_>>>()
+            .ok_or_else(|| error!("Found uninitialized bytes: `{:?}`", self.bytes))?;
+        read_target_uint(&raw)
+    }
+
+    pub fn read_uint(&self) -> Result<u128, Error> {
+        if self.bytes.len() > 16 {
+            return Err(error!("Allocation is bigger than largest integer"));
+        }
+        let raw = self.raw_bytes()?;
+        read_target_uint(&raw)
+    }
+
+    pub fn read_int(&self) -> Result<i128, Error> {
+        if self.bytes.len() > 16 {
+            return Err(error!("Allocation is bigger than largest integer"));
+        }
+        let raw = self.raw_bytes()?;
+        read_target_int(&raw)
+    }
+
+    pub fn read_bool(&self) -> Result<bool, Error> {
+        match self.read_int()? {
+            0 => Ok(false),
+            1 => Ok(true),
+            val @ _ => Err(error!("Unexpected value for bool: `{val}`")),
+        }
+    }
+
+    pub fn is_null(&self) -> Result<bool, Error> {
+        let len = self.bytes.len();
+        let ptr_len = MachineInfo::target_pointer_width().bytes();
+        if len != ptr_len {
+            return Err(error!("Expected width of pointer (`{ptr_len}`), but found: `{len}`"));
+        }
+        Ok(self.read_uint()? == 0)
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ConstantKind {
     Allocated(Allocation),
-- 
cgit 1.4.1-3-g733a5


From 9cb6463af7a9a67b8fdcd2c846b42310f4245e98 Mon Sep 17 00:00:00 2001
From: "Celina G. Val" <celinval@amazon.com>
Date: Wed, 6 Dec 2023 21:33:49 -0800
Subject: Fix conversion to StaticDef and add test

---
 compiler/stable_mir/src/compiler_interface.rs    |  2 +-
 compiler/stable_mir/src/mir/mono.rs              |  2 +-
 tests/ui-fulldeps/stable-mir/check_allocation.rs | 16 ++++++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs
index 928f320c2fa..17c5212fb9c 100644
--- a/compiler/stable_mir/src/compiler_interface.rs
+++ b/compiler/stable_mir/src/compiler_interface.rs
@@ -162,7 +162,7 @@ pub trait Context {
     fn krate(&self, def_id: DefId) -> Crate;
     fn instance_name(&self, def: InstanceDef, trimmed: bool) -> Symbol;
 
-    /// Return the number of bytes for a pointer size.
+    /// Return information about the target machine.
     fn target_info(&self) -> MachineInfo;
 }
 
diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs
index acf268e56b3..bc5d4a3b8f4 100644
--- a/compiler/stable_mir/src/mir/mono.rs
+++ b/compiler/stable_mir/src/mir/mono.rs
@@ -220,7 +220,7 @@ impl TryFrom<CrateItem> for StaticDef {
     type Error = crate::Error;
 
     fn try_from(value: CrateItem) -> Result<Self, Self::Error> {
-        if matches!(value.kind(), ItemKind::Static | ItemKind::Const) {
+        if matches!(value.kind(), ItemKind::Static) {
             Ok(StaticDef(value.0))
         } else {
             Err(Error::new(format!("Expected a static item, but found: {value:?}")))
diff --git a/tests/ui-fulldeps/stable-mir/check_allocation.rs b/tests/ui-fulldeps/stable-mir/check_allocation.rs
index 170b1fd73b1..5051d7b7d82 100644
--- a/tests/ui-fulldeps/stable-mir/check_allocation.rs
+++ b/tests/ui-fulldeps/stable-mir/check_allocation.rs
@@ -40,6 +40,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
     let items = stable_mir::all_local_items();
     check_foo(*get_item(&items, (ItemKind::Static, "FOO")).unwrap());
     check_bar(*get_item(&items, (ItemKind::Static, "BAR")).unwrap());
+    check_len(*get_item(&items, (ItemKind::Static, "LEN")).unwrap());
     ControlFlow::Continue(())
 }
 
@@ -76,6 +77,19 @@ fn check_bar(item: CrateItem) {
     assert_eq!(allocation.bytes[0].unwrap(), Char::CapitalB.to_u8());
     assert_eq!(allocation.bytes[1].unwrap(), Char::SmallA.to_u8());
     assert_eq!(allocation.bytes[2].unwrap(), Char::SmallR.to_u8());
+    assert_eq!(std::str::from_utf8(&allocation.raw_bytes().unwrap()), Ok("Bar"));
+}
+
+/// Check the allocation data for `LEN`.
+///
+/// ```no_run
+/// static LEN: usize = 2;
+/// ```
+fn check_len(item: CrateItem) {
+    let def = StaticDef::try_from(item).unwrap();
+    let alloc = def.eval_initializer().unwrap();
+    assert!(alloc.provenance.ptrs.is_empty());
+    assert_eq!(alloc.read_uint(), Ok(2));
 }
 
 // Use internal API to find a function in a crate.
@@ -109,11 +123,13 @@ fn generate_input(path: &str) -> std::io::Result<()> {
     write!(
         file,
         r#"
+    static LEN: usize = 2;
     static FOO: [&str; 2] = ["hi", "there"];
     static BAR: &str = "Bar";
 
     pub fn main() {{
         println!("{{FOO:?}}! {{BAR}}");
+        assert_eq!(FOO.len(), LEN);
     }}"#
     )?;
     Ok(())
-- 
cgit 1.4.1-3-g733a5


From 0a0e7e6c0d9627dc6a48e7e0522abd9f7964564f Mon Sep 17 00:00:00 2001
From: "Celina G. Val" <celinval@amazon.com>
Date: Thu, 7 Dec 2023 13:15:43 -0800
Subject: Add tests to allocation methods and fix is_null()

---
 compiler/stable_mir/src/mir/body.rs              | 20 ++++-
 compiler/stable_mir/src/ty.rs                    |  6 +-
 tests/ui-fulldeps/stable-mir/check_allocation.rs | 95 ++++++++++++++++++++++--
 3 files changed, 114 insertions(+), 7 deletions(-)

diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index d03ded20e9b..663275d9a0f 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -21,7 +21,7 @@ pub struct Body {
     pub(super) arg_count: usize,
 
     /// Debug information pertaining to user variables, including captures.
-    pub(super) var_debug_info: Vec<VarDebugInfo>,
+    pub var_debug_info: Vec<VarDebugInfo>,
 }
 
 pub type BasicBlockIdx = usize;
@@ -616,6 +616,24 @@ pub struct VarDebugInfo {
     pub argument_index: Option<u16>,
 }
 
+impl VarDebugInfo {
+    /// Return a local variable if this info is related to one.
+    pub fn local(&self) -> Option<Local> {
+        match &self.value {
+            VarDebugInfoContents::Place(place) if place.projection.is_empty() => Some(place.local),
+            VarDebugInfoContents::Place(_) | VarDebugInfoContents::Const(_) => None,
+        }
+    }
+
+    /// Return a constant if this info is related to one.
+    pub fn constant(&self) -> Option<&ConstOperand> {
+        match &self.value {
+            VarDebugInfoContents::Place(_) => None,
+            VarDebugInfoContents::Const(const_op) => Some(const_op),
+        }
+    }
+}
+
 pub type SourceScope = u32;
 
 #[derive(Clone, Debug, Eq, PartialEq)]
diff --git a/compiler/stable_mir/src/ty.rs b/compiler/stable_mir/src/ty.rs
index 4caac575c1e..bea7702bd34 100644
--- a/compiler/stable_mir/src/ty.rs
+++ b/compiler/stable_mir/src/ty.rs
@@ -901,6 +901,7 @@ impl Allocation {
         read_target_uint(&raw)
     }
 
+    /// Read this allocation and try to convert it to an unassigned integer.
     pub fn read_uint(&self) -> Result<u128, Error> {
         if self.bytes.len() > 16 {
             return Err(error!("Allocation is bigger than largest integer"));
@@ -909,6 +910,7 @@ impl Allocation {
         read_target_uint(&raw)
     }
 
+    /// Read this allocation and try to convert it to a signed integer.
     pub fn read_int(&self) -> Result<i128, Error> {
         if self.bytes.len() > 16 {
             return Err(error!("Allocation is bigger than largest integer"));
@@ -917,6 +919,7 @@ impl Allocation {
         read_target_int(&raw)
     }
 
+    /// Read this allocation and try to convert it to a boolean.
     pub fn read_bool(&self) -> Result<bool, Error> {
         match self.read_int()? {
             0 => Ok(false),
@@ -925,13 +928,14 @@ impl Allocation {
         }
     }
 
+    /// Read this allocation as a pointer and return whether it represents a `null` pointer.
     pub fn is_null(&self) -> Result<bool, Error> {
         let len = self.bytes.len();
         let ptr_len = MachineInfo::target_pointer_width().bytes();
         if len != ptr_len {
             return Err(error!("Expected width of pointer (`{ptr_len}`), but found: `{len}`"));
         }
-        Ok(self.read_uint()? == 0)
+        Ok(self.read_uint()? == 0 && self.provenance.ptrs.is_empty())
     }
 }
 
diff --git a/tests/ui-fulldeps/stable-mir/check_allocation.rs b/tests/ui-fulldeps/stable-mir/check_allocation.rs
index 5051d7b7d82..d2bc58be4c1 100644
--- a/tests/ui-fulldeps/stable-mir/check_allocation.rs
+++ b/tests/ui-fulldeps/stable-mir/check_allocation.rs
@@ -23,12 +23,16 @@ extern crate stable_mir;
 
 use rustc_middle::ty::TyCtxt;
 use rustc_smir::rustc_internal;
-use stable_mir::{CrateItem, CrateItems, ItemKind};
 use stable_mir::crate_def::CrateDef;
 use stable_mir::mir::alloc::GlobalAlloc;
-use stable_mir::mir::mono::StaticDef;
+use stable_mir::mir::mono::{Instance, StaticDef};
+use stable_mir::mir::Body;
+use stable_mir::ty::{Allocation, ConstantKind};
+use stable_mir::{CrateItem, CrateItems, ItemKind};
 use std::ascii::Char;
 use std::assert_matches::assert_matches;
+use std::cmp::{max, min};
+use std::collections::HashMap;
 use std::io::Write;
 use std::ops::ControlFlow;
 
@@ -41,6 +45,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
     check_foo(*get_item(&items, (ItemKind::Static, "FOO")).unwrap());
     check_bar(*get_item(&items, (ItemKind::Static, "BAR")).unwrap());
     check_len(*get_item(&items, (ItemKind::Static, "LEN")).unwrap());
+    check_other_consts(*get_item(&items, (ItemKind::Fn, "other_consts")).unwrap());
     ControlFlow::Continue(())
 }
 
@@ -80,6 +85,73 @@ fn check_bar(item: CrateItem) {
     assert_eq!(std::str::from_utf8(&allocation.raw_bytes().unwrap()), Ok("Bar"));
 }
 
+/// Check the allocation data for constants used in `other_consts` function.
+fn check_other_consts(item: CrateItem) {
+    // Instance body will force constant evaluation.
+    let body = Instance::try_from(item).unwrap().body().unwrap();
+    let assigns = collect_consts(&body);
+    assert_eq!(assigns.len(), 9);
+    for (name, alloc) in assigns {
+        match name.as_str() {
+            "_max_u128" => {
+                assert_eq!(alloc.read_uint(), Ok(u128::MAX), "Failed parsing allocation: {alloc:?}")
+            }
+            "_min_i128" => {
+                assert_eq!(alloc.read_int(), Ok(i128::MIN), "Failed parsing allocation: {alloc:?}")
+            }
+            "_max_i8" => {
+                assert_eq!(
+                    alloc.read_int().unwrap() as i8,
+                    i8::MAX,
+                    "Failed parsing allocation: {alloc:?}"
+                )
+            }
+            "_char" => {
+                assert_eq!(
+                    char::from_u32(alloc.read_uint().unwrap() as u32),
+                    Some('x'),
+                    "Failed parsing allocation: {alloc:?}"
+                )
+            }
+            "_false" => {
+                assert_eq!(alloc.read_bool(), Ok(false), "Failed parsing allocation: {alloc:?}")
+            }
+            "_true" => {
+                assert_eq!(alloc.read_bool(), Ok(true), "Failed parsing allocation: {alloc:?}")
+            }
+            "_ptr" => {
+                assert_eq!(alloc.is_null(), Ok(false), "Failed parsing allocation: {alloc:?}")
+            }
+            "_null_ptr" => {
+                assert_eq!(alloc.is_null(), Ok(true), "Failed parsing allocation: {alloc:?}")
+            }
+            "_tuple" => {
+                // The order of fields is not guaranteed.
+                let first = alloc.read_partial_uint(0..4).unwrap();
+                let second = alloc.read_partial_uint(4..8).unwrap();
+                assert_eq!(max(first, second) as u32, u32::MAX);
+                assert_eq!(min(first, second), 10);
+            }
+            _ => {
+                unreachable!("{name} -- {alloc:?}")
+            }
+        }
+    }
+}
+
+/// Collects all the constant assignments.
+pub fn collect_consts(body: &Body) -> HashMap<String, &Allocation> {
+    body.var_debug_info
+        .iter()
+        .filter_map(|info| {
+            info.constant().map(|const_op| {
+                let ConstantKind::Allocated(alloc) = const_op.const_.kind() else { unreachable!() };
+                (info.name.clone(), alloc)
+            })
+        })
+        .collect::<HashMap<_, _>>()
+}
+
 /// Check the allocation data for `LEN`.
 ///
 /// ```no_run
@@ -97,9 +169,7 @@ fn get_item<'a>(
     items: &'a CrateItems,
     item: (ItemKind, &str),
 ) -> Option<&'a stable_mir::CrateItem> {
-    items.iter().find(|crate_item| {
-        (item.0 == crate_item.kind()) && crate_item.name() == item.1
-    })
+    items.iter().find(|crate_item| (item.0 == crate_item.kind()) && crate_item.name() == item.1)
 }
 
 /// This test will generate and analyze a dummy crate using the stable mir.
@@ -126,10 +196,25 @@ fn generate_input(path: &str) -> std::io::Result<()> {
     static LEN: usize = 2;
     static FOO: [&str; 2] = ["hi", "there"];
     static BAR: &str = "Bar";
+    const NULL: *const u8 = std::ptr::null();
+    const TUPLE: (u32, u32) = (10, u32::MAX);
+
+    fn other_consts() {{
+        let _max_u128 = u128::MAX;
+        let _min_i128 = i128::MIN;
+        let _max_i8 = i8::MAX;
+        let _char = 'x';
+        let _false = false;
+        let _true = true;
+        let _ptr = &BAR;
+        let _null_ptr: *const u8 = NULL;
+        let _tuple = TUPLE;
+    }}
 
     pub fn main() {{
         println!("{{FOO:?}}! {{BAR}}");
         assert_eq!(FOO.len(), LEN);
+        other_consts();
     }}"#
     )?;
     Ok(())
-- 
cgit 1.4.1-3-g733a5


From 9e15c49ae3bf675f7f7240da14bfd03338d62dc7 Mon Sep 17 00:00:00 2001
From: "Celina G. Val" <celinval@amazon.com>
Date: Thu, 7 Dec 2023 14:35:58 -0800
Subject: Add a test to evaluate type intrinsic.

---
 tests/ui-fulldeps/stable-mir/check_allocation.rs | 39 ++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/tests/ui-fulldeps/stable-mir/check_allocation.rs b/tests/ui-fulldeps/stable-mir/check_allocation.rs
index d2bc58be4c1..88c41537d9f 100644
--- a/tests/ui-fulldeps/stable-mir/check_allocation.rs
+++ b/tests/ui-fulldeps/stable-mir/check_allocation.rs
@@ -25,9 +25,9 @@ use rustc_middle::ty::TyCtxt;
 use rustc_smir::rustc_internal;
 use stable_mir::crate_def::CrateDef;
 use stable_mir::mir::alloc::GlobalAlloc;
-use stable_mir::mir::mono::{Instance, StaticDef};
-use stable_mir::mir::Body;
-use stable_mir::ty::{Allocation, ConstantKind};
+use stable_mir::mir::mono::{Instance, InstanceKind, StaticDef};
+use stable_mir::mir::{Body, TerminatorKind};
+use stable_mir::ty::{Allocation, ConstantKind, RigidTy, TyKind};
 use stable_mir::{CrateItem, CrateItems, ItemKind};
 use std::ascii::Char;
 use std::assert_matches::assert_matches;
@@ -46,6 +46,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
     check_bar(*get_item(&items, (ItemKind::Static, "BAR")).unwrap());
     check_len(*get_item(&items, (ItemKind::Static, "LEN")).unwrap());
     check_other_consts(*get_item(&items, (ItemKind::Fn, "other_consts")).unwrap());
+    check_type_id(*get_item(&items, (ItemKind::Fn, "check_type_id")).unwrap());
     ControlFlow::Continue(())
 }
 
@@ -139,6 +140,30 @@ fn check_other_consts(item: CrateItem) {
     }
 }
 
+/// Check that we can retrieve the type id of char and bool, and that they have different values.
+fn check_type_id(item: CrateItem) {
+    let body = Instance::try_from(item).unwrap().body().unwrap();
+    let mut ids: Vec<u128> = vec![];
+    for term in body.blocks.iter().map(|bb| &bb.terminator) {
+        match &term.kind {
+            TerminatorKind::Call { func, destination, .. } => {
+                let TyKind::RigidTy(ty) = func.ty(body.locals()).unwrap().kind() else {
+                    unreachable!()
+                };
+                let RigidTy::FnDef(def, args) = ty else { unreachable!() };
+                let instance = Instance::resolve(def, &args).unwrap();
+                assert_eq!(instance.kind, InstanceKind::Intrinsic);
+                let dest_ty = destination.ty(body.locals()).unwrap();
+                let alloc = instance.try_const_eval(dest_ty).unwrap();
+                ids.push(alloc.read_uint().unwrap());
+            }
+            _ => { /* Do nothing */ }
+        }
+    }
+    assert_eq!(ids.len(), 2);
+    assert_ne!(ids[0], ids[1]);
+}
+
 /// Collects all the constant assignments.
 pub fn collect_consts(body: &Body) -> HashMap<String, &Allocation> {
     body.var_debug_info
@@ -193,6 +218,9 @@ fn generate_input(path: &str) -> std::io::Result<()> {
     write!(
         file,
         r#"
+    #![feature(core_intrinsics)]
+    use std::intrinsics::type_id;
+
     static LEN: usize = 2;
     static FOO: [&str; 2] = ["hi", "there"];
     static BAR: &str = "Bar";
@@ -211,6 +239,11 @@ fn generate_input(path: &str) -> std::io::Result<()> {
         let _tuple = TUPLE;
     }}
 
+    fn check_type_id() {{
+        let _char_id = type_id::<char>();
+        let _bool_id = type_id::<bool>();
+    }}
+
     pub fn main() {{
         println!("{{FOO:?}}! {{BAR}}");
         assert_eq!(FOO.len(), LEN);
-- 
cgit 1.4.1-3-g733a5


From ac50f4b5719921b139161c298f81d43367ccd127 Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Fri, 8 Dec 2023 02:16:29 +0100
Subject: update fixme

---
 compiler/rustc_infer/src/infer/combine.rs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs
index cfd53fc71eb..bab21bc237a 100644
--- a/compiler/rustc_infer/src/infer/combine.rs
+++ b/compiler/rustc_infer/src/infer/combine.rs
@@ -512,10 +512,9 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
             } else {
                 match a_ty.kind() {
                     &ty::Alias(ty::AliasKind::Projection, data) => {
-                        // FIXME: This does not handle subtyping correctly, we should switch to
-                        // alias-relate in the new solver and could instead create a new inference
-                        // variable for `a_ty`, emitting `Projection(a_ty, a_infer)` and
-                        // `a_infer <: b_ty`.
+                        // FIXME: This does not handle subtyping correctly, we could
+                        // instead create a new inference variable for `a_ty`, emitting
+                        // `Projection(a_ty, a_infer)` and `a_infer <: b_ty`.
                         self.obligations.push(Obligation::new(
                             self.tcx(),
                             self.trace.cause.clone(),
-- 
cgit 1.4.1-3-g733a5


From dabedb711f13e14401e6be2f9a58a9bc1974a488 Mon Sep 17 00:00:00 2001
From: oksbsb <oksbsb@126.com>
Date: Thu, 7 Dec 2023 22:39:07 +0800
Subject: 1. fix jobserver GLOBAL_CLIENT_CHECKED uninitialized before use 2.
 jobserver::initialize_checked should call before build_session, still should
 use EarlyErrorHandler, so revert stderr change in #118635

---
 compiler/rustc_data_structures/src/jobserver.rs    |  2 +-
 compiler/rustc_interface/src/interface.rs          |  4 ++++
 compiler/rustc_interface/src/tests.rs              |  2 ++
 compiler/rustc_session/src/session.rs              | 23 +++++++++++-----------
 .../run-make/jobserver-error/cannot_open_fd.stderr |  2 --
 tests/run-make/jobserver-error/not_a_pipe.stderr   |  2 --
 6 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/compiler/rustc_data_structures/src/jobserver.rs b/compiler/rustc_data_structures/src/jobserver.rs
index b777bfd4d3c..412e33aaa65 100644
--- a/compiler/rustc_data_structures/src/jobserver.rs
+++ b/compiler/rustc_data_structures/src/jobserver.rs
@@ -52,7 +52,7 @@ fn default_client() -> Client {
 
 static GLOBAL_CLIENT_CHECKED: OnceLock<Client> = OnceLock::new();
 
-pub fn check(report_warning: impl FnOnce(&'static str)) {
+pub fn initialize_checked(report_warning: impl FnOnce(&'static str)) {
     let client_checked = match &*GLOBAL_CLIENT {
         Ok(client) => client.clone(),
         Err(e) => {
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 7831b251db4..6527e87d396 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -316,6 +316,10 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
     // Set parallel mode before thread pool creation, which will create `Lock`s.
     rustc_data_structures::sync::set_dyn_thread_safe_mode(config.opts.unstable_opts.threads > 1);
 
+    // Check jobserver before run_in_thread_pool_with_globals, which call jobserver::acquire_thread
+    let early_handler = EarlyErrorHandler::new(config.opts.error_format);
+    early_handler.initialize_checked_jobserver();
+
     util::run_in_thread_pool_with_globals(
         config.opts.edition,
         config.opts.unstable_opts.threads,
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index f7b6ab331a5..ce58b2ab061 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -27,6 +27,8 @@ use std::sync::Arc;
 
 fn mk_session(matches: getopts::Matches) -> (Session, Cfg) {
     let mut early_handler = EarlyErrorHandler::new(ErrorOutputType::default());
+    early_handler.initialize_checked_jobserver();
+
     let registry = registry::Registry::new(&[]);
     let sessopts = build_session_options(&mut early_handler, &matches);
     let temps_dir = sessopts.unstable_opts.temps_dir.as_deref().map(PathBuf::from);
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 123e9c788f5..24c7459392a 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -1474,17 +1474,6 @@ pub fn build_session(
     let asm_arch =
         if target_cfg.allow_asm { InlineAsmArch::from_str(&target_cfg.arch).ok() } else { None };
 
-    // Check jobserver before getting `jobserver::client`.
-    jobserver::check(|err| {
-        #[allow(rustc::untranslatable_diagnostic)]
-        #[allow(rustc::diagnostic_outside_of_impl)]
-        parse_sess
-            .span_diagnostic
-            .struct_warn(err)
-            .note("the build environment is likely misconfigured")
-            .emit()
-    });
-
     let sess = Session {
         target: target_cfg,
         host,
@@ -1792,6 +1781,18 @@ impl EarlyErrorHandler {
     pub fn early_warn(&self, msg: impl Into<DiagnosticMessage>) {
         self.handler.struct_warn(msg).emit()
     }
+
+    pub fn initialize_checked_jobserver(&self) {
+        // initialize jobserver before getting `jobserver::client` and `build_session`.
+        jobserver::initialize_checked(|err| {
+            #[allow(rustc::untranslatable_diagnostic)]
+            #[allow(rustc::diagnostic_outside_of_impl)]
+            self.handler
+                .struct_warn(err)
+                .note("the build environment is likely misconfigured")
+                .emit()
+        });
+    }
 }
 
 fn mk_emitter(output: ErrorOutputType) -> Box<DynEmitter> {
diff --git a/tests/run-make/jobserver-error/cannot_open_fd.stderr b/tests/run-make/jobserver-error/cannot_open_fd.stderr
index a2f77a94e4f..343de5cd52c 100644
--- a/tests/run-make/jobserver-error/cannot_open_fd.stderr
+++ b/tests/run-make/jobserver-error/cannot_open_fd.stderr
@@ -4,5 +4,3 @@ warning: failed to connect to jobserver from environment variable `MAKEFLAGS="--
 
 error: no input filename given
 
-warning: 1 warning emitted
-
diff --git a/tests/run-make/jobserver-error/not_a_pipe.stderr b/tests/run-make/jobserver-error/not_a_pipe.stderr
index 9158fda6e47..536c04576b9 100644
--- a/tests/run-make/jobserver-error/not_a_pipe.stderr
+++ b/tests/run-make/jobserver-error/not_a_pipe.stderr
@@ -2,5 +2,3 @@ warning: failed to connect to jobserver from environment variable `MAKEFLAGS="--
   |
   = note: the build environment is likely misconfigured
 
-warning: 1 warning emitted
-
-- 
cgit 1.4.1-3-g733a5


From eb53721a34d1d910b900aa753e0a00dc72ef41ac Mon Sep 17 00:00:00 2001
From: jyn <github@jyn.dev>
Date: Thu, 7 Dec 2023 22:54:41 -0500
Subject: recurse into refs when comparing tys for diagnostics

---
 .../rustc_infer/src/infer/error_reporting/mod.rs   | 94 ++++++++++++----------
 .../associated-const-impl-wrong-lifetime.stderr    |  4 +-
 .../dont-suggest-cyclic-constraint.stderr          |  4 +-
 .../async-example-desugared-boxed-in-trait.stderr  |  4 +-
 .../regions-bound-missing-bound-in-impl.stderr     |  8 +-
 tests/ui/box/issue-82446.stderr                    |  4 +-
 .../expect-fn-supply-fn.stderr                     | 12 +--
 tests/ui/closures/multiple-fn-bounds.stderr        |  4 +-
 tests/ui/fn/fn-pointer-mismatch.rs                 | 12 +--
 tests/ui/fn/fn-pointer-mismatch.stderr             | 16 ++--
 .../ui/generic-associated-types/issue-88360.stderr |  4 +-
 ...subtype.bound_a_b_ret_a_vs_bound_a_ret_a.stderr |  4 +-
 .../subtype/hr-subtype.bound_a_vs_free_x.stderr    |  4 +-
 .../trait-bounds/hrtb-exists-forall-fn.stderr      |  4 +-
 .../in-trait/specialization-broken.stderr          |  4 +-
 ...-alias-impl-trait-declaration-too-subtle.stderr |  8 +-
 tests/ui/implied-bounds/issue-100690.stderr        |  4 +-
 tests/ui/issues/issue-17905-2.stderr               |  8 +-
 tests/ui/issues/issue-20225.stderr                 | 12 +--
 tests/ui/issues/issue-24322.stderr                 |  4 +-
 tests/ui/issues/issue-37884.stderr                 |  4 +-
 .../let-else-binding-explicit-mut-annotated.stderr |  8 +-
 tests/ui/lifetimes/issue-79187-2.stderr            |  8 +-
 tests/ui/lub-glb/old-lub-glb-hr-noteq1.leak.stderr |  4 +-
 .../ui/lub-glb/old-lub-glb-hr-noteq1.noleak.stderr |  4 +-
 tests/ui/lub-glb/old-lub-glb-hr-noteq2.leak.stderr |  4 +-
 .../mir/field-projection-mutating-context.stderr   |  4 +-
 .../closure-arg-type-mismatch.stderr               |  4 +-
 tests/ui/mismatched_types/issue-36053-2.stderr     |  4 +-
 tests/ui/mismatched_types/normalize-fn-sig.stderr  |  4 +-
 ...f-clone-call-on-ref-due-to-missing-bound.stderr |  4 +-
 tests/ui/nll/relate_tys/hr-fn-aaa-as-aba.stderr    |  8 +-
 tests/ui/nll/relate_tys/universe-violation.stderr  |  4 +-
 tests/ui/nll/trait-associated-constant.stderr      |  4 +-
 tests/ui/or-patterns/inconsistent-modes.stderr     |  4 +-
 .../issue-73553-misinterp-range-literal.stderr     |  4 +-
 tests/ui/regions/issue-101280.stderr               |  4 +-
 ...gion-lifetime-bounds-on-fns-where-clause.stderr |  4 +-
 ...iple-lifetime-bounds-on-fns-where-clause.stderr |  4 +-
 .../regions/regions-lifetime-bounds-on-fns.stderr  |  4 +-
 .../resolve-inconsistent-binding-mode.stderr       |  4 +-
 .../rfcs/rfc-2005-default-binding-mode/lit.stderr  |  4 +-
 .../issue-92010-trait-bound-not-satisfied.stderr   |  4 +-
 tests/ui/static/static-reference-to-fn-1.stderr    |  4 +-
 tests/ui/suggestions/as-ref.stderr                 | 12 +--
 ...one-on-unconstrained-borrowed-type-param.stderr |  4 +-
 .../method-access-to-range-literal-typo.stderr     |  4 +-
 tests/ui/suggestions/mut-ref-reassignment.stderr   |  4 +-
 tests/ui/traits/impl-method-mismatch.stderr        |  4 +-
 tests/ui/traits/wrong-mul-method-signature.stderr  |  4 +-
 .../ui/type/type-check/point-at-inference-2.stderr |  8 +-
 tests/ui/type/type-mismatch.stderr                 | 24 +++---
 tests/ui/typeck/bad-index-due-to-nested.stderr     |  8 +-
 tests/ui/typeck/mismatched-map-under-self.stderr   |  4 +-
 tests/ui/ufcs/ufcs-explicit-self-bad.rs            |  8 +-
 tests/ui/ufcs/ufcs-explicit-self-bad.stderr        | 20 ++---
 tests/ui/unsafe/unsafe-trait-impl.rs               |  4 +-
 tests/ui/unsafe/unsafe-trait-impl.stderr           |  4 +-
 58 files changed, 224 insertions(+), 214 deletions(-)

diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
index ecaf9f4e169..745c3d195ad 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
@@ -1181,37 +1181,54 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         debug!("cmp(t1={}, t1.kind={:?}, t2={}, t2.kind={:?})", t1, t1.kind(), t2, t2.kind());
 
         // helper functions
-        fn equals<'tcx>(a: Ty<'tcx>, b: Ty<'tcx>) -> bool {
-            match (a.kind(), b.kind()) {
-                (a, b) if *a == *b => true,
-                (&ty::Int(_), &ty::Infer(ty::InferTy::IntVar(_)))
-                | (
-                    &ty::Infer(ty::InferTy::IntVar(_)),
-                    &ty::Int(_) | &ty::Infer(ty::InferTy::IntVar(_)),
-                )
-                | (&ty::Float(_), &ty::Infer(ty::InferTy::FloatVar(_)))
-                | (
-                    &ty::Infer(ty::InferTy::FloatVar(_)),
-                    &ty::Float(_) | &ty::Infer(ty::InferTy::FloatVar(_)),
-                ) => true,
-                _ => false,
+        let recurse = |t1, t2, values: &mut (DiagnosticStyledString, DiagnosticStyledString)| {
+            let (x1, x2) = self.cmp(t1, t2);
+            (values.0).0.extend(x1.0);
+            (values.1).0.extend(x2.0);
+        };
+
+        fn fmt_region<'tcx>(region: ty::Region<'tcx>) -> String {
+            let mut r = region.to_string();
+            if r == "'_" {
+                r.clear();
+            } else {
+                r.push(' ');
             }
+            format!("&{r}")
         }
 
-        fn push_ty_ref<'tcx>(
+        fn push_ref<'tcx>(
             region: ty::Region<'tcx>,
-            ty: Ty<'tcx>,
             mutbl: hir::Mutability,
             s: &mut DiagnosticStyledString,
         ) {
-            let mut r = region.to_string();
-            if r == "'_" {
-                r.clear();
+            s.push_highlighted(fmt_region(region));
+            s.push_highlighted(mutbl.prefix_str());
+        }
+
+        fn cmp_ty_refs<'tcx>(
+            r1: ty::Region<'tcx>,
+            mut1: hir::Mutability,
+            r2: ty::Region<'tcx>,
+            mut2: hir::Mutability,
+            ss: &mut (DiagnosticStyledString, DiagnosticStyledString),
+        ) {
+            let (r1, r2) = (fmt_region(r1), fmt_region(r2));
+            if r1 != r2 {
+                ss.0.push_highlighted(r1);
+                ss.1.push_highlighted(r2);
             } else {
-                r.push(' ');
+                ss.0.push_normal(r1);
+                ss.1.push_normal(r2);
+            }
+
+            if mut1 != mut2 {
+                ss.0.push_highlighted(mut1.prefix_str());
+                ss.1.push_highlighted(mut2.prefix_str());
+            } else {
+                ss.0.push_normal(mut1.prefix_str());
+                ss.1.push_normal(mut2.prefix_str());
             }
-            s.push_highlighted(format!("&{}{}", r, mutbl.prefix_str()));
-            s.push_normal(ty.to_string());
         }
 
         // process starts here
@@ -1310,9 +1327,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                             values.0.push_normal("_");
                             values.1.push_normal("_");
                         } else {
-                            let (x1, x2) = self.cmp(ta1, ta2);
-                            (values.0).0.extend(x1.0);
-                            (values.1).0.extend(x2.0);
+                            recurse(ta1, ta2, &mut values);
                         }
                         self.push_comma(&mut values.0, &mut values.1, len, i);
                     }
@@ -1418,27 +1433,24 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 }
             }
 
-            // When finding T != &T, highlight only the borrow
-            (&ty::Ref(r1, ref_ty1, mutbl1), _) if equals(ref_ty1, t2) => {
+            // When finding `&T != &T`, compare the references, then recurse into pointee type
+            (&ty::Ref(r1, ref_ty1, mutbl1), &ty::Ref(r2, ref_ty2, mutbl2)) => {
                 let mut values = (DiagnosticStyledString::new(), DiagnosticStyledString::new());
-                push_ty_ref(r1, ref_ty1, mutbl1, &mut values.0);
-                values.1.push_normal(t2.to_string());
+                cmp_ty_refs(r1, mutbl1, r2, mutbl2, &mut values);
+                recurse(ref_ty1, ref_ty2, &mut values);
                 values
             }
-            (_, &ty::Ref(r2, ref_ty2, mutbl2)) if equals(t1, ref_ty2) => {
+            // When finding T != &T, highlight the borrow
+            (&ty::Ref(r1, ref_ty1, mutbl1), _) => {
                 let mut values = (DiagnosticStyledString::new(), DiagnosticStyledString::new());
-                values.0.push_normal(t1.to_string());
-                push_ty_ref(r2, ref_ty2, mutbl2, &mut values.1);
+                push_ref(r1, mutbl1, &mut values.0);
+                recurse(ref_ty1, t2, &mut values);
                 values
             }
-
-            // When encountering &T != &mut T, highlight only the borrow
-            (&ty::Ref(r1, ref_ty1, mutbl1), &ty::Ref(r2, ref_ty2, mutbl2))
-                if equals(ref_ty1, ref_ty2) =>
-            {
+            (_, &ty::Ref(r2, ref_ty2, mutbl2)) => {
                 let mut values = (DiagnosticStyledString::new(), DiagnosticStyledString::new());
-                push_ty_ref(r1, ref_ty1, mutbl1, &mut values.0);
-                push_ty_ref(r2, ref_ty2, mutbl2, &mut values.1);
+                push_ref(r2, mutbl2, &mut values.1);
+                recurse(t1, ref_ty2, &mut values);
                 values
             }
 
@@ -1448,9 +1460,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     (DiagnosticStyledString::normal("("), DiagnosticStyledString::normal("("));
                 let len = args1.len();
                 for (i, (left, right)) in args1.iter().zip(args2).enumerate() {
-                    let (x1, x2) = self.cmp(left, right);
-                    (values.0).0.extend(x1.0);
-                    (values.1).0.extend(x2.0);
+                    recurse(left, right, &mut values);
                     self.push_comma(&mut values.0, &mut values.1, len, i);
                 }
                 if len == 1 {
diff --git a/tests/ui/associated-consts/associated-const-impl-wrong-lifetime.stderr b/tests/ui/associated-consts/associated-const-impl-wrong-lifetime.stderr
index 6037122a365..1304bef4211 100644
--- a/tests/ui/associated-consts/associated-const-impl-wrong-lifetime.stderr
+++ b/tests/ui/associated-consts/associated-const-impl-wrong-lifetime.stderr
@@ -4,8 +4,8 @@ error[E0308]: const not compatible with trait
 LL |     const NAME: &'a str = "unit";
    |     ^^^^^^^^^^^^^^^^^^^ lifetime mismatch
    |
-   = note: expected reference `&'static str`
-              found reference `&'a str`
+   = note: expected reference `&'static _`
+              found reference `&'a _`
 note: the lifetime `'a` as defined here...
   --> $DIR/associated-const-impl-wrong-lifetime.rs:6:6
    |
diff --git a/tests/ui/associated-types/dont-suggest-cyclic-constraint.stderr b/tests/ui/associated-types/dont-suggest-cyclic-constraint.stderr
index 606084aea34..1b4694a2125 100644
--- a/tests/ui/associated-types/dont-suggest-cyclic-constraint.stderr
+++ b/tests/ui/associated-types/dont-suggest-cyclic-constraint.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |     debug_assert_eq!(iter.next(), Some(value));
    |                                   ^^^^^^^^^^^ expected `Option<<I as Iterator>::Item>`, found `Option<&<I as Iterator>::Item>`
    |
-   = note: expected enum `Option<<I as Iterator>::Item>`
-              found enum `Option<&<I as Iterator>::Item>`
+   = note: expected enum `Option<_>`
+              found enum `Option<&_>`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/async-await/in-trait/async-example-desugared-boxed-in-trait.stderr b/tests/ui/async-await/in-trait/async-example-desugared-boxed-in-trait.stderr
index 34aded73da5..54df0edf5a8 100644
--- a/tests/ui/async-await/in-trait/async-example-desugared-boxed-in-trait.stderr
+++ b/tests/ui/async-await/in-trait/async-example-desugared-boxed-in-trait.stderr
@@ -9,8 +9,8 @@ note: type in trait
    |
 LL |     fn foo(&self) -> Pin<Box<dyn Future<Output = i32> + '_>>;
    |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   = note: expected signature `fn(&i32) -> Pin<Box<dyn Future<Output = i32>>>`
-              found signature `fn(&i32) -> impl Future<Output = i32>`
+   = note: expected signature `fn(&_) -> Pin<Box<dyn Future<Output = i32>>>`
+              found signature `fn(&_) -> impl Future<Output = i32>`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/borrowck/regions-bound-missing-bound-in-impl.stderr b/tests/ui/borrowck/regions-bound-missing-bound-in-impl.stderr
index 7ebea3c03d3..54d8f26f4ea 100644
--- a/tests/ui/borrowck/regions-bound-missing-bound-in-impl.stderr
+++ b/tests/ui/borrowck/regions-bound-missing-bound-in-impl.stderr
@@ -22,8 +22,8 @@ error[E0308]: method not compatible with trait
 LL |     fn wrong_bound1<'b,'c,'d:'a+'c>(self, b: Inv<'b>, c: Inv<'c>, d: Inv<'d>) {
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ lifetime mismatch
    |
-   = note: expected signature `fn(&'a isize, Inv<'c>, Inv<'c>, Inv<'_>)`
-              found signature `fn(&'a isize, Inv<'_>, Inv<'c>, Inv<'_>)`
+   = note: expected signature `fn(&'a _, Inv<'c>, Inv<'c>, Inv<'_>)`
+              found signature `fn(&'a _, Inv<'_>, Inv<'c>, Inv<'_>)`
 note: the lifetime `'c` as defined here...
   --> $DIR/regions-bound-missing-bound-in-impl.rs:27:24
    |
@@ -41,8 +41,8 @@ error[E0308]: method not compatible with trait
 LL |     fn wrong_bound1<'b,'c,'d:'a+'c>(self, b: Inv<'b>, c: Inv<'c>, d: Inv<'d>) {
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ lifetime mismatch
    |
-   = note: expected signature `fn(&'a isize, Inv<'c>, Inv<'c>, Inv<'_>)`
-              found signature `fn(&'a isize, Inv<'_>, Inv<'c>, Inv<'_>)`
+   = note: expected signature `fn(&'a _, Inv<'c>, Inv<'c>, Inv<'_>)`
+              found signature `fn(&'a _, Inv<'_>, Inv<'c>, Inv<'_>)`
 note: the lifetime `'c` as defined here...
   --> $DIR/regions-bound-missing-bound-in-impl.rs:27:24
    |
diff --git a/tests/ui/box/issue-82446.stderr b/tests/ui/box/issue-82446.stderr
index 568d23c2cb7..66daaceffb1 100644
--- a/tests/ui/box/issue-82446.stderr
+++ b/tests/ui/box/issue-82446.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |         val
    |         ^^^ expected `Box<dyn MyTrait>`, found `&Box<dyn MyTrait>`
    |
-   = note: expected struct `Box<(dyn MyTrait + 'static)>`
-           found reference `&Box<(dyn MyTrait + 'static)>`
+   = note: expected struct `Box<_>`
+           found reference `&Box<_>`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/closure-expected-type/expect-fn-supply-fn.stderr b/tests/ui/closure-expected-type/expect-fn-supply-fn.stderr
index e6ddc606897..e010f0502f8 100644
--- a/tests/ui/closure-expected-type/expect-fn-supply-fn.stderr
+++ b/tests/ui/closure-expected-type/expect-fn-supply-fn.stderr
@@ -25,8 +25,8 @@ error[E0308]: mismatched types
 LL |     with_closure_expecting_fn_with_free_region(|x: fn(&u32), y| {});
    |                                                 ^ one type is more general than the other
    |
-   = note: expected fn pointer `fn(&u32)`
-              found fn pointer `for<'a> fn(&'a u32)`
+   = note: expected fn pointer `fn(&_)`
+              found fn pointer `for<'a> fn(&'a _)`
 
 error[E0308]: mismatched types
   --> $DIR/expect-fn-supply-fn.rs:39:50
@@ -34,8 +34,8 @@ error[E0308]: mismatched types
 LL |     with_closure_expecting_fn_with_bound_region(|x: fn(&'x u32), y| {});
    |                                                  ^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'a> fn(&'a u32)`
-              found fn pointer `fn(&u32)`
+   = note: expected fn pointer `for<'a> fn(&'a _)`
+              found fn pointer `fn(&_)`
 
 error[E0308]: mismatched types
   --> $DIR/expect-fn-supply-fn.rs:48:50
@@ -43,8 +43,8 @@ error[E0308]: mismatched types
 LL |     with_closure_expecting_fn_with_bound_region(|x: Foo<'_>, y| {
    |                                                  ^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'a> fn(&'a u32)`
-              found fn pointer `fn(&u32)`
+   = note: expected fn pointer `for<'a> fn(&'a _)`
+              found fn pointer `fn(&_)`
 
 error: aborting due to 5 previous errors
 
diff --git a/tests/ui/closures/multiple-fn-bounds.stderr b/tests/ui/closures/multiple-fn-bounds.stderr
index d510fc585f6..325652ef14c 100644
--- a/tests/ui/closures/multiple-fn-bounds.stderr
+++ b/tests/ui/closures/multiple-fn-bounds.stderr
@@ -6,8 +6,8 @@ LL |     foo(move |x| v);
    |     |
    |     expected due to this
    |
-   = note: expected closure signature `fn(char) -> _`
-              found closure signature `for<'a> fn(&'a char) -> _`
+   = note: expected closure signature `fn(_) -> _`
+              found closure signature `for<'a> fn(&'a _) -> _`
 note: closure inferred to have a different signature due to this bound
   --> $DIR/multiple-fn-bounds.rs:1:11
    |
diff --git a/tests/ui/fn/fn-pointer-mismatch.rs b/tests/ui/fn/fn-pointer-mismatch.rs
index 0597478cb42..1c50d8b0f8b 100644
--- a/tests/ui/fn/fn-pointer-mismatch.rs
+++ b/tests/ui/fn/fn-pointer-mismatch.rs
@@ -35,20 +35,20 @@ fn main() {
     // suggest removing reference
     let c: fn(u32) -> u32 = &foo;
     //~^ ERROR mismatched types
-    //~| expected fn pointer `fn(u32) -> u32`
-    //~| found reference `&fn(u32) -> u32 {foo}`
+    //~| expected fn pointer `fn(_) -> _`
+    //~| found reference `&fn(_) -> _ {foo}`
 
     // suggest using reference
     let d: &fn(u32) -> u32 = foo;
     //~^ ERROR mismatched types
-    //~| expected reference `&fn(u32) -> u32`
-    //~| found fn item `fn(u32) -> u32 {foo}`
+    //~| expected reference `&fn(_) -> _`
+    //~| found fn item `fn(_) -> _ {foo}`
 
     // suggest casting with reference
     let e: &fn(u32) -> u32 = &foo;
     //~^ ERROR mismatched types
-    //~| expected reference `&fn(u32) -> u32`
-    //~| found reference `&fn(u32) -> u32 {foo}`
+    //~| expected reference `&fn(_) -> _`
+    //~| found reference `&fn(_) -> _ {foo}`
 
     // OK
     let mut z: fn(u32) -> u32 = foo as fn(u32) -> u32;
diff --git a/tests/ui/fn/fn-pointer-mismatch.stderr b/tests/ui/fn/fn-pointer-mismatch.stderr
index 87ece845b83..9cda11639d0 100644
--- a/tests/ui/fn/fn-pointer-mismatch.stderr
+++ b/tests/ui/fn/fn-pointer-mismatch.stderr
@@ -6,8 +6,8 @@ LL |     let g = if n % 2 == 0 { &foo } else { &bar };
    |                             |
    |                             expected because of this
    |
-   = note: expected reference `&fn(u32) -> u32 {foo}`
-              found reference `&fn(u32) -> u32 {bar}`
+   = note: expected reference `&fn(_) -> _ {foo}`
+              found reference `&fn(_) -> _ {bar}`
    = note: different fn items have unique types, even if their signatures are the same
    = help: consider casting both fn items to fn pointers using `as fn(u32) -> u32`
 
@@ -47,8 +47,8 @@ LL |     let c: fn(u32) -> u32 = &foo;
    |            |
    |            expected due to this
    |
-   = note: expected fn pointer `fn(u32) -> u32`
-               found reference `&fn(u32) -> u32 {foo}`
+   = note: expected fn pointer `fn(_) -> _`
+               found reference `&fn(_) -> _ {foo}`
 help: consider removing the reference
    |
 LL |     let c: fn(u32) -> u32 = foo;
@@ -62,8 +62,8 @@ LL |     let d: &fn(u32) -> u32 = foo;
    |            |
    |            expected due to this
    |
-   = note: expected reference `&fn(u32) -> u32`
-                found fn item `fn(u32) -> u32 {foo}`
+   = note: expected reference `&fn(_) -> _`
+                found fn item `fn(_) -> _ {foo}`
 help: consider using a reference
    |
 LL |     let d: &fn(u32) -> u32 = &foo;
@@ -77,8 +77,8 @@ LL |     let e: &fn(u32) -> u32 = &foo;
    |            |
    |            expected due to this
    |
-   = note: expected reference `&fn(u32) -> u32`
-              found reference `&fn(u32) -> u32 {foo}`
+   = note: expected reference `&fn(_) -> _`
+              found reference `&fn(_) -> _ {foo}`
    = note: fn items are distinct from fn pointers
 help: consider casting to a fn pointer
    |
diff --git a/tests/ui/generic-associated-types/issue-88360.stderr b/tests/ui/generic-associated-types/issue-88360.stderr
index 64cd55e684c..49d36acadd6 100644
--- a/tests/ui/generic-associated-types/issue-88360.stderr
+++ b/tests/ui/generic-associated-types/issue-88360.stderr
@@ -9,8 +9,8 @@ LL |     fn copy(&self) -> Self::Gat<'_> where T: Copy {
 LL |         *self.test()
    |         ^^^^^^^^^^^^ expected `&T`, found type parameter `T`
    |
-   = note:   expected reference `&T`
-           found type parameter `T`
+   = note:   expected reference `&_`
+           found type parameter `_`
 help: consider removing deref here
    |
 LL -         *self.test()
diff --git a/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_b_ret_a_vs_bound_a_ret_a.stderr b/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_b_ret_a_vs_bound_a_ret_a.stderr
index 7cb7edfafeb..d7f0860a026 100644
--- a/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_b_ret_a_vs_bound_a_ret_a.stderr
+++ b/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_b_ret_a_vs_bound_a_ret_a.stderr
@@ -8,8 +8,8 @@ LL | / check! { bound_a_b_ret_a_vs_bound_a_ret_a: (for<'a,'b> fn(&'a u32, &'b u3
 LL | | for<'a>    fn(&'a u32, &'a u32) -> &'a u32) }
    | |_____________________________________________- in this macro invocation
    |
-   = note: expected enum `Option<for<'a, 'b> fn(&'a u32, &'b u32) -> &'a u32>`
-              found enum `Option<for<'a> fn(&'a u32, &'a u32) -> &'a u32>`
+   = note: expected enum `Option<for<'a, 'b> fn(&'a _, &'b _) -> &'a _>`
+              found enum `Option<for<'a> fn(&'a _, &'a _) -> &'a _>`
    = note: this error originates in the macro `check` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: aborting due to 1 previous error
diff --git a/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_vs_free_x.stderr b/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_vs_free_x.stderr
index c6adbd91e78..9b5ca3b2056 100644
--- a/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_vs_free_x.stderr
+++ b/tests/ui/higher-ranked/subtype/hr-subtype.bound_a_vs_free_x.stderr
@@ -8,8 +8,8 @@ LL | / check! { bound_a_vs_free_x: (for<'a> fn(&'a u32),
 LL | | fn(&'x u32)) }
    | |______________- in this macro invocation
    |
-   = note: expected enum `Option<for<'a> fn(&'a u32)>`
-              found enum `Option<fn(&u32)>`
+   = note: expected enum `Option<for<'a> fn(&'a _)>`
+              found enum `Option<fn(&_)>`
    = note: this error originates in the macro `check` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: aborting due to 1 previous error
diff --git a/tests/ui/higher-ranked/trait-bounds/hrtb-exists-forall-fn.stderr b/tests/ui/higher-ranked/trait-bounds/hrtb-exists-forall-fn.stderr
index f269babcf71..2d6b8063f69 100644
--- a/tests/ui/higher-ranked/trait-bounds/hrtb-exists-forall-fn.stderr
+++ b/tests/ui/higher-ranked/trait-bounds/hrtb-exists-forall-fn.stderr
@@ -6,8 +6,8 @@ LL |     let _: for<'b> fn(&'b u32) = foo();
    |            |
    |            expected due to this
    |
-   = note: expected fn pointer `for<'b> fn(&'b u32)`
-              found fn pointer `fn(&u32)`
+   = note: expected fn pointer `for<'b> fn(&'b _)`
+              found fn pointer `fn(&_)`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/impl-trait/in-trait/specialization-broken.stderr b/tests/ui/impl-trait/in-trait/specialization-broken.stderr
index 25c0adeddbd..b8a8e2401b2 100644
--- a/tests/ui/impl-trait/in-trait/specialization-broken.stderr
+++ b/tests/ui/impl-trait/in-trait/specialization-broken.stderr
@@ -15,8 +15,8 @@ note: type in trait
    |
 LL |     fn bar(&self) -> impl Sized;
    |                      ^^^^^^^^^^
-   = note: expected signature `fn(&U) -> impl Sized`
-              found signature `fn(&U) -> U`
+   = note: expected signature `fn(&_) -> impl Sized`
+              found signature `fn(&_) -> U`
 
 error: method with return-position `impl Trait` in trait cannot be specialized
   --> $DIR/specialization-broken.rs:15:5
diff --git a/tests/ui/impl-trait/recursive-type-alias-impl-trait-declaration-too-subtle.stderr b/tests/ui/impl-trait/recursive-type-alias-impl-trait-declaration-too-subtle.stderr
index fe765271bd2..07ac1a37e75 100644
--- a/tests/ui/impl-trait/recursive-type-alias-impl-trait-declaration-too-subtle.stderr
+++ b/tests/ui/impl-trait/recursive-type-alias-impl-trait-declaration-too-subtle.stderr
@@ -18,8 +18,8 @@ LL |         fn eq(&self, _other: &(Foo, i32)) -> bool {
    |                              expected `a::Bar`, found opaque type
    |                              help: change the parameter type to match the trait: `&(a::Bar, i32)`
    |
-   = note: expected signature `fn(&a::Bar, &(a::Bar, i32)) -> _`
-              found signature `fn(&a::Bar, &(a::Foo, i32)) -> _`
+   = note: expected signature `fn(&a::Bar, &(a::Bar, _)) -> _`
+              found signature `fn(&a::Bar, &(a::Foo, _)) -> _`
 
 error: unconstrained opaque type
   --> $DIR/recursive-type-alias-impl-trait-declaration-too-subtle.rs:18:16
@@ -41,8 +41,8 @@ LL |         fn eq(&self, _other: &(Bar, i32)) -> bool {
    |                              expected opaque type, found `b::Bar`
    |                              help: change the parameter type to match the trait: `&(b::Foo, i32)`
    |
-   = note: expected signature `fn(&b::Bar, &(b::Foo, i32)) -> _`
-              found signature `fn(&b::Bar, &(b::Bar, i32)) -> _`
+   = note: expected signature `fn(&b::Bar, &(b::Foo, _)) -> _`
+              found signature `fn(&b::Bar, &(b::Bar, _)) -> _`
 note: this item must have the opaque type in its signature in order to be able to register hidden types
   --> $DIR/recursive-type-alias-impl-trait-declaration-too-subtle.rs:24:12
    |
diff --git a/tests/ui/implied-bounds/issue-100690.stderr b/tests/ui/implied-bounds/issue-100690.stderr
index 49f2fcd0a67..df069d875ce 100644
--- a/tests/ui/implied-bounds/issue-100690.stderr
+++ b/tests/ui/implied-bounds/issue-100690.stderr
@@ -6,8 +6,8 @@ LL |         real_dispatch(f)
    |         |
    |         required by a bound introduced by this call
    |
-   = note: expected a closure with arguments `(&mut UIView<'a, T>,)`
-              found a closure with arguments `(&mut UIView<'_, T>,)`
+   = note: expected a closure with arguments `(&mut UIView<'a, _>,)`
+              found a closure with arguments `(&mut UIView<'_, _>,)`
 note: required by a bound in `real_dispatch`
   --> $DIR/issue-100690.rs:9:8
    |
diff --git a/tests/ui/issues/issue-17905-2.stderr b/tests/ui/issues/issue-17905-2.stderr
index 88b5fbec6cf..c66cb222489 100644
--- a/tests/ui/issues/issue-17905-2.stderr
+++ b/tests/ui/issues/issue-17905-2.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched `self` parameter type
 LL |     fn say(self: &Pair<&str, isize>) {
    |                  ^^^^^^^^^^^^^^^^^^ lifetime mismatch
    |
-   = note: expected struct `Pair<&str, _>`
-              found struct `Pair<&str, _>`
+   = note: expected struct `Pair<&_, _>`
+              found struct `Pair<&_, _>`
 note: the anonymous lifetime defined here...
   --> $DIR/issue-17905-2.rs:8:24
    |
@@ -23,8 +23,8 @@ error[E0308]: mismatched `self` parameter type
 LL |     fn say(self: &Pair<&str, isize>) {
    |                  ^^^^^^^^^^^^^^^^^^ lifetime mismatch
    |
-   = note: expected struct `Pair<&str, _>`
-              found struct `Pair<&str, _>`
+   = note: expected struct `Pair<&_, _>`
+              found struct `Pair<&_, _>`
 note: the anonymous lifetime as defined here...
   --> $DIR/issue-17905-2.rs:5:5
    |
diff --git a/tests/ui/issues/issue-20225.stderr b/tests/ui/issues/issue-20225.stderr
index b34aa8e1ff5..2d24a5bbd50 100644
--- a/tests/ui/issues/issue-20225.stderr
+++ b/tests/ui/issues/issue-20225.stderr
@@ -9,8 +9,8 @@ LL |   extern "rust-call" fn call(&self, (_,): (T,)) {}
    |                                           expected `&'a T`, found type parameter `T`
    |                                           help: change the parameter type to match the trait: `(&'a T,)`
    |
-   = note: expected signature `extern "rust-call" fn(&Foo, (&'a T,))`
-              found signature `extern "rust-call" fn(&Foo, (T,))`
+   = note: expected signature `extern "rust-call" fn(&Foo, (&'a _,))`
+              found signature `extern "rust-call" fn(&Foo, (_,))`
 
 error[E0053]: method `call_mut` has an incompatible type for trait
   --> $DIR/issue-20225.rs:11:51
@@ -23,8 +23,8 @@ LL |   extern "rust-call" fn call_mut(&mut self, (_,): (T,)) {}
    |                                                   expected `&'a T`, found type parameter `T`
    |                                                   help: change the parameter type to match the trait: `(&'a T,)`
    |
-   = note: expected signature `extern "rust-call" fn(&mut Foo, (&'a T,))`
-              found signature `extern "rust-call" fn(&mut Foo, (T,))`
+   = note: expected signature `extern "rust-call" fn(&mut Foo, (&'a _,))`
+              found signature `extern "rust-call" fn(&mut Foo, (_,))`
 
 error[E0053]: method `call_once` has an incompatible type for trait
   --> $DIR/issue-20225.rs:18:47
@@ -38,8 +38,8 @@ LL |   extern "rust-call" fn call_once(self, (_,): (T,)) {}
    |                                               expected `&'a T`, found type parameter `T`
    |                                               help: change the parameter type to match the trait: `(&'a T,)`
    |
-   = note: expected signature `extern "rust-call" fn(Foo, (&'a T,))`
-              found signature `extern "rust-call" fn(Foo, (T,))`
+   = note: expected signature `extern "rust-call" fn(Foo, (&'a _,))`
+              found signature `extern "rust-call" fn(Foo, (_,))`
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/issues/issue-24322.stderr b/tests/ui/issues/issue-24322.stderr
index b260d027388..d078069addc 100644
--- a/tests/ui/issues/issue-24322.stderr
+++ b/tests/ui/issues/issue-24322.stderr
@@ -6,8 +6,8 @@ LL |     let x: &fn(&B) -> u32 = &B::func;
    |            |
    |            expected due to this
    |
-   = note: expected reference `&for<'a> fn(&'a B) -> u32`
-              found reference `&for<'a> fn(&'a B) -> u32 {B::func}`
+   = note: expected reference `&for<'a> fn(&'a B) -> _`
+              found reference `&for<'a> fn(&'a B) -> _ {B::func}`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/issues/issue-37884.stderr b/tests/ui/issues/issue-37884.stderr
index 633abeb6f22..b7c0095d682 100644
--- a/tests/ui/issues/issue-37884.stderr
+++ b/tests/ui/issues/issue-37884.stderr
@@ -4,8 +4,8 @@ error[E0308]: method not compatible with trait
 LL |     fn next(&'a mut self) -> Option<Self::Item>
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ lifetime mismatch
    |
-   = note: expected signature `fn(&mut RepeatMut<'a, T>) -> Option<_>`
-              found signature `fn(&'a mut RepeatMut<'a, T>) -> Option<_>`
+   = note: expected signature `fn(&mut RepeatMut<'_, _>) -> Option<_>`
+              found signature `fn(&'a mut RepeatMut<'_, _>) -> Option<_>`
 note: the anonymous lifetime as defined here...
   --> $DIR/issue-37884.rs:6:5
    |
diff --git a/tests/ui/let-else/let-else-binding-explicit-mut-annotated.stderr b/tests/ui/let-else/let-else-binding-explicit-mut-annotated.stderr
index 065787cab08..1d41335036e 100644
--- a/tests/ui/let-else/let-else-binding-explicit-mut-annotated.stderr
+++ b/tests/ui/let-else/let-else-binding-explicit-mut-annotated.stderr
@@ -6,8 +6,8 @@ LL |     let Some(n): &mut Option<i32> = &&Some(5i32) else { return };
    |                  |
    |                  expected due to this
    |
-   = note: expected mutable reference `&mut Option<i32>`
-                      found reference `&&Option<i32>`
+   = note: expected mutable reference `&mut Option<_>`
+                      found reference `&&Option<_>`
 
 error[E0308]: mismatched types
   --> $DIR/let-else-binding-explicit-mut-annotated.rs:13:37
@@ -17,8 +17,8 @@ LL |     let Some(n): &mut Option<i32> = &&mut Some(5i32) else { return };
    |                  |
    |                  expected due to this
    |
-   = note: expected mutable reference `&mut Option<i32>`
-                      found reference `&&mut Option<i32>`
+   = note: expected mutable reference `&mut Option<_>`
+                      found reference `&&mut Option<_>`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/lifetimes/issue-79187-2.stderr b/tests/ui/lifetimes/issue-79187-2.stderr
index 86a4ac4132e..e8115bb6b06 100644
--- a/tests/ui/lifetimes/issue-79187-2.stderr
+++ b/tests/ui/lifetimes/issue-79187-2.stderr
@@ -54,8 +54,8 @@ error[E0308]: mismatched types
 LL |     take_foo(|a: &i32| a);
    |     ^^^^^^^^^^^^^^^^^^^^^ one type is more general than the other
    |
-   = note: expected reference `&i32`
-              found reference `&i32`
+   = note: expected reference `&_`
+              found reference `&_`
 note: the lifetime requirement is introduced here
   --> $DIR/issue-79187-2.rs:5:21
    |
@@ -68,8 +68,8 @@ error[E0308]: mismatched types
 LL |     take_foo(|a: &i32| -> &i32 { a });
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ one type is more general than the other
    |
-   = note: expected reference `&i32`
-              found reference `&i32`
+   = note: expected reference `&_`
+              found reference `&_`
 note: the lifetime requirement is introduced here
   --> $DIR/issue-79187-2.rs:5:21
    |
diff --git a/tests/ui/lub-glb/old-lub-glb-hr-noteq1.leak.stderr b/tests/ui/lub-glb/old-lub-glb-hr-noteq1.leak.stderr
index c3d7960de56..10e3fc92868 100644
--- a/tests/ui/lub-glb/old-lub-glb-hr-noteq1.leak.stderr
+++ b/tests/ui/lub-glb/old-lub-glb-hr-noteq1.leak.stderr
@@ -12,8 +12,8 @@ LL | |
 LL | |     };
    | |_____- `match` arms have incompatible types
    |
-   = note: expected fn pointer `for<'a, 'b> fn(&'a u8, &'b u8) -> &'a u8`
-              found fn pointer `for<'a> fn(&'a u8, &'a u8) -> &'a u8`
+   = note: expected fn pointer `for<'a, 'b> fn(&'a _, &'b _) -> &'a _`
+              found fn pointer `for<'a> fn(&'a _, &'a _) -> &'a _`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/lub-glb/old-lub-glb-hr-noteq1.noleak.stderr b/tests/ui/lub-glb/old-lub-glb-hr-noteq1.noleak.stderr
index 8e4a514c7c6..bf77875fa7a 100644
--- a/tests/ui/lub-glb/old-lub-glb-hr-noteq1.noleak.stderr
+++ b/tests/ui/lub-glb/old-lub-glb-hr-noteq1.noleak.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |         _ => y,
    |              ^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'a, 'b> fn(&'a u8, &'b u8) -> &'a u8`
-              found fn pointer `for<'a> fn(&'a u8, &'a u8) -> &'a u8`
+   = note: expected fn pointer `for<'a, 'b> fn(&'a _, &'b _) -> &'a _`
+              found fn pointer `for<'a> fn(&'a _, &'a _) -> &'a _`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/lub-glb/old-lub-glb-hr-noteq2.leak.stderr b/tests/ui/lub-glb/old-lub-glb-hr-noteq2.leak.stderr
index a1958cc436a..1c8925e0bfb 100644
--- a/tests/ui/lub-glb/old-lub-glb-hr-noteq2.leak.stderr
+++ b/tests/ui/lub-glb/old-lub-glb-hr-noteq2.leak.stderr
@@ -11,8 +11,8 @@ LL | |
 LL | |     };
    | |_____- `match` arms have incompatible types
    |
-   = note: expected fn pointer `for<'a> fn(&'a u8, &'a u8) -> &'a u8`
-              found fn pointer `for<'a, 'b> fn(&'a u8, &'b u8) -> &'a u8`
+   = note: expected fn pointer `for<'a> fn(&'a _, &'a _) -> &'a _`
+              found fn pointer `for<'a, 'b> fn(&'a _, &'b _) -> &'a _`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/mir/field-projection-mutating-context.stderr b/tests/ui/mir/field-projection-mutating-context.stderr
index 62c9e55a44b..c7289c0f07c 100644
--- a/tests/ui/mir/field-projection-mutating-context.stderr
+++ b/tests/ui/mir/field-projection-mutating-context.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |     let Foo(ref mut y): Foo<fn(&'static str)> = x;
    |             ^^^^^^^^^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'a> fn(&'a str)`
-              found fn pointer `fn(&str)`
+   = note: expected fn pointer `for<'a> fn(&'a _)`
+              found fn pointer `fn(&_)`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/mismatched_types/closure-arg-type-mismatch.stderr b/tests/ui/mismatched_types/closure-arg-type-mismatch.stderr
index 760e3327b77..e9808b86991 100644
--- a/tests/ui/mismatched_types/closure-arg-type-mismatch.stderr
+++ b/tests/ui/mismatched_types/closure-arg-type-mismatch.stderr
@@ -6,8 +6,8 @@ LL |     a.iter().map(|_: (u32, u32)| 45);
    |              |
    |              expected due to this
    |
-   = note: expected closure signature `fn(&(u32, u32)) -> _`
-              found closure signature `fn((u32, u32)) -> _`
+   = note: expected closure signature `fn(&(_, _)) -> _`
+              found closure signature `fn((_, _)) -> _`
 note: required by a bound in `map`
   --> $SRC_DIR/core/src/iter/traits/iterator.rs:LL:COL
 help: consider adjusting the signature so it borrows its argument
diff --git a/tests/ui/mismatched_types/issue-36053-2.stderr b/tests/ui/mismatched_types/issue-36053-2.stderr
index bac27788a2d..292525daa3d 100644
--- a/tests/ui/mismatched_types/issue-36053-2.stderr
+++ b/tests/ui/mismatched_types/issue-36053-2.stderr
@@ -6,8 +6,8 @@ LL |     once::<&str>("str").fuse().filter(|a: &str| true).count();
    |                                |
    |                                expected due to this
    |
-   = note: expected closure signature `for<'a> fn(&'a &str) -> _`
-              found closure signature `for<'a> fn(&'a str) -> _`
+   = note: expected closure signature `for<'a> fn(&'a &_) -> _`
+              found closure signature `for<'a> fn(&'a _) -> _`
 note: required by a bound in `filter`
   --> $SRC_DIR/core/src/iter/traits/iterator.rs:LL:COL
 help: consider adjusting the signature so it borrows its argument
diff --git a/tests/ui/mismatched_types/normalize-fn-sig.stderr b/tests/ui/mismatched_types/normalize-fn-sig.stderr
index 2166de85f1f..0fb3975c3c2 100644
--- a/tests/ui/mismatched_types/normalize-fn-sig.stderr
+++ b/tests/ui/mismatched_types/normalize-fn-sig.stderr
@@ -6,8 +6,8 @@ LL |     needs_i32_ref_fn(foo::<()>);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected fn pointer `fn(&'static i32, i32)`
-                 found fn item `fn(i32, &'static i32) {foo::<()>}`
+   = note: expected fn pointer `fn(&'static _, _)`
+                 found fn item `fn(_, &'static _) {foo::<()>}`
 note: function defined here
   --> $DIR/normalize-fn-sig.rs:11:4
    |
diff --git a/tests/ui/moves/assignment-of-clone-call-on-ref-due-to-missing-bound.stderr b/tests/ui/moves/assignment-of-clone-call-on-ref-due-to-missing-bound.stderr
index 821661f1a56..6a9d76f7998 100644
--- a/tests/ui/moves/assignment-of-clone-call-on-ref-due-to-missing-bound.stderr
+++ b/tests/ui/moves/assignment-of-clone-call-on-ref-due-to-missing-bound.stderr
@@ -6,8 +6,8 @@ LL |             let mut x: HashSet<Day> = v.clone();
    |                        |
    |                        expected due to this
    |
-   = note: expected struct `HashSet<Day>`
-           found reference `&HashSet<Day>`
+   = note: expected struct `HashSet<_>`
+           found reference `&HashSet<_>`
 note: `HashSet<Day>` does not implement `Clone`, so `&HashSet<Day>` was cloned instead
   --> $DIR/assignment-of-clone-call-on-ref-due-to-missing-bound.rs:18:39
    |
diff --git a/tests/ui/nll/relate_tys/hr-fn-aaa-as-aba.stderr b/tests/ui/nll/relate_tys/hr-fn-aaa-as-aba.stderr
index 7d76c916d6d..58fd7776f2f 100644
--- a/tests/ui/nll/relate_tys/hr-fn-aaa-as-aba.stderr
+++ b/tests/ui/nll/relate_tys/hr-fn-aaa-as-aba.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |     let a: for<'a, 'b> fn(&'a u32, &'b u32) -> &'a u32 = make_it();
    |                                                          ^^^^^^^^^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'a, 'b> fn(&'a u32, &'b u32) -> &'a u32`
-              found fn pointer `for<'a> fn(&'a u32, &'a u32) -> &'a u32`
+   = note: expected fn pointer `for<'a, 'b> fn(&'a _, &'b _) -> &'a _`
+              found fn pointer `for<'a> fn(&'a _, &'a _) -> &'a _`
 
 error[E0308]: mismatched types
   --> $DIR/hr-fn-aaa-as-aba.rs:20:12
@@ -13,8 +13,8 @@ error[E0308]: mismatched types
 LL |     let _: for<'a, 'b> fn(&'a u32, &'b u32) -> &'a u32 = make_it();
    |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'a, 'b> fn(&'a u32, &'b u32) -> &'a u32`
-              found fn pointer `for<'a> fn(&'a u32, &'a u32) -> &'a u32`
+   = note: expected fn pointer `for<'a, 'b> fn(&'a _, &'b _) -> &'a _`
+              found fn pointer `for<'a> fn(&'a _, &'a _) -> &'a _`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/nll/relate_tys/universe-violation.stderr b/tests/ui/nll/relate_tys/universe-violation.stderr
index b585eee0769..a538a59a495 100644
--- a/tests/ui/nll/relate_tys/universe-violation.stderr
+++ b/tests/ui/nll/relate_tys/universe-violation.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |     let b: fn(&u32) -> &u32 = a;
    |                               ^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'a> fn(&'a u32) -> &'a u32`
-              found fn pointer `fn(&u32) -> &u32`
+   = note: expected fn pointer `for<'a> fn(&'a _) -> &'a _`
+              found fn pointer `fn(&_) -> &_`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/nll/trait-associated-constant.stderr b/tests/ui/nll/trait-associated-constant.stderr
index f6277508eeb..371f7860d4d 100644
--- a/tests/ui/nll/trait-associated-constant.stderr
+++ b/tests/ui/nll/trait-associated-constant.stderr
@@ -4,8 +4,8 @@ error[E0308]: const not compatible with trait
 LL |     const AC: Option<&'c str> = None;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^ lifetime mismatch
    |
-   = note: expected enum `Option<&'b str>`
-              found enum `Option<&'c str>`
+   = note: expected enum `Option<&'b _>`
+              found enum `Option<&'c _>`
 note: the lifetime `'c` as defined here...
   --> $DIR/trait-associated-constant.rs:20:18
    |
diff --git a/tests/ui/or-patterns/inconsistent-modes.stderr b/tests/ui/or-patterns/inconsistent-modes.stderr
index f6367ef8234..19618d336f8 100644
--- a/tests/ui/or-patterns/inconsistent-modes.stderr
+++ b/tests/ui/or-patterns/inconsistent-modes.stderr
@@ -57,8 +57,8 @@ LL |     let (Ok(ref a) | Err(ref mut a)): Result<&u8, &mut u8> = Ok(&0);
    |             |            types differ in mutability
    |             first introduced with type `&&u8` here
    |
-   = note:      expected reference `&&u8`
-           found mutable reference `&mut &mut u8`
+   = note:      expected reference `&&_`
+           found mutable reference `&mut &mut _`
    = note: a binding must have the same type in all alternatives
 
 error[E0308]: mismatched types
diff --git a/tests/ui/range/issue-73553-misinterp-range-literal.stderr b/tests/ui/range/issue-73553-misinterp-range-literal.stderr
index 52efa241d0b..15e55708c37 100644
--- a/tests/ui/range/issue-73553-misinterp-range-literal.stderr
+++ b/tests/ui/range/issue-73553-misinterp-range-literal.stderr
@@ -6,8 +6,8 @@ LL |     demo(tell(1)..tell(10));
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected reference `&std::ops::Range<usize>`
-                 found struct `std::ops::Range<usize>`
+   = note: expected reference `&std::ops::Range<_>`
+                 found struct `std::ops::Range<_>`
 note: function defined here
   --> $DIR/issue-73553-misinterp-range-literal.rs:3:4
    |
diff --git a/tests/ui/regions/issue-101280.stderr b/tests/ui/regions/issue-101280.stderr
index 70953808b81..48deb99494b 100644
--- a/tests/ui/regions/issue-101280.stderr
+++ b/tests/ui/regions/issue-101280.stderr
@@ -6,8 +6,8 @@ LL | fn f<'r>(f: fn(Cell<(&'r i32, &i32)>)) -> Ty {
 LL |     f
    |     ^ one type is more general than the other
    |
-   = note: expected fn pointer `for<'r> fn(Cell<(&'r i32, &'r i32)>)`
-              found fn pointer `for<'a> fn(Cell<(&'r i32, &'a i32)>)`
+   = note: expected fn pointer `for<'r> fn(Cell<(&'r _, &'r _)>)`
+              found fn pointer `for<'a> fn(Cell<(&'r _, &'a _)>)`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/regions/region-lifetime-bounds-on-fns-where-clause.stderr b/tests/ui/regions/region-lifetime-bounds-on-fns-where-clause.stderr
index 03d5a0be723..d8269514bef 100644
--- a/tests/ui/regions/region-lifetime-bounds-on-fns-where-clause.stderr
+++ b/tests/ui/regions/region-lifetime-bounds-on-fns-where-clause.stderr
@@ -6,8 +6,8 @@ LL |     let _: fn(&mut &isize, &mut &isize) = a;
    |            |
    |            expected due to this
    |
-   = note: expected fn pointer `for<'a, 'b, 'c, 'd> fn(&'a mut &'b isize, &'c mut &'d isize)`
-                 found fn item `for<'a, 'b> fn(&'a mut &isize, &'b mut &isize) {a::<'_, '_>}`
+   = note: expected fn pointer `for<'a, 'b, 'c, 'd> fn(&'a mut &'b _, &'c mut &'d _)`
+                 found fn item `for<'a, 'b> fn(&'a mut &_, &'b mut &_) {a::<'_, '_>}`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/regions/region-multiple-lifetime-bounds-on-fns-where-clause.stderr b/tests/ui/regions/region-multiple-lifetime-bounds-on-fns-where-clause.stderr
index 250571f1556..e383f352b9e 100644
--- a/tests/ui/regions/region-multiple-lifetime-bounds-on-fns-where-clause.stderr
+++ b/tests/ui/regions/region-multiple-lifetime-bounds-on-fns-where-clause.stderr
@@ -6,8 +6,8 @@ LL |     let _: fn(&mut &isize, &mut &isize, &mut &isize) = a;
    |            |
    |            expected due to this
    |
-   = note: expected fn pointer `for<'a, 'b, 'c, 'd, 'e, 'f> fn(&'a mut &'b isize, &'c mut &'d isize, &'e mut &'f isize)`
-                 found fn item `for<'a, 'b, 'c> fn(&'a mut &isize, &'b mut &isize, &'c mut &isize) {a::<'_, '_, '_>}`
+   = note: expected fn pointer `for<'a, 'b, 'c, 'd, 'e, 'f> fn(&'a mut &'b _, &'c mut &'d _, &'e mut &'f _)`
+                 found fn item `for<'a, 'b, 'c> fn(&'a mut &_, &'b mut &_, &'c mut &_) {a::<'_, '_, '_>}`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/regions/regions-lifetime-bounds-on-fns.stderr b/tests/ui/regions/regions-lifetime-bounds-on-fns.stderr
index 31dd7efe067..989e91c702b 100644
--- a/tests/ui/regions/regions-lifetime-bounds-on-fns.stderr
+++ b/tests/ui/regions/regions-lifetime-bounds-on-fns.stderr
@@ -6,8 +6,8 @@ LL |     let _: fn(&mut &isize, &mut &isize) = a;
    |            |
    |            expected due to this
    |
-   = note: expected fn pointer `for<'a, 'b, 'c, 'd> fn(&'a mut &'b isize, &'c mut &'d isize)`
-                 found fn item `for<'a, 'b> fn(&'a mut &isize, &'b mut &isize) {a::<'_, '_>}`
+   = note: expected fn pointer `for<'a, 'b, 'c, 'd> fn(&'a mut &'b _, &'c mut &'d _)`
+                 found fn item `for<'a, 'b> fn(&'a mut &_, &'b mut &_) {a::<'_, '_>}`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/resolve/resolve-inconsistent-binding-mode.stderr b/tests/ui/resolve/resolve-inconsistent-binding-mode.stderr
index c805c9eb125..6e44c280f75 100644
--- a/tests/ui/resolve/resolve-inconsistent-binding-mode.stderr
+++ b/tests/ui/resolve/resolve-inconsistent-binding-mode.stderr
@@ -62,8 +62,8 @@ LL |         Opts::A(ref mut i) | Opts::B(ref i) => {}
    |                 |
    |                 first introduced with type `&mut isize` here
    |
-   = note: expected mutable reference `&mut isize`
-                      found reference `&isize`
+   = note: expected mutable reference `&mut _`
+                      found reference `&_`
    = note: in the same arm, a binding must have the same type in all alternatives
 
 error: aborting due to 6 previous errors
diff --git a/tests/ui/rfcs/rfc-2005-default-binding-mode/lit.stderr b/tests/ui/rfcs/rfc-2005-default-binding-mode/lit.stderr
index 181f57899a9..970a9c151c1 100644
--- a/tests/ui/rfcs/rfc-2005-default-binding-mode/lit.stderr
+++ b/tests/ui/rfcs/rfc-2005-default-binding-mode/lit.stderr
@@ -6,8 +6,8 @@ LL |     match &s {
 LL |             "abc" => true,
    |             ^^^^^ expected `&&str`, found `&str`
    |
-   = note: expected reference `&&str`
-              found reference `&'static str`
+   = note: expected reference `&&_`
+              found reference `&'static _`
 
 error[E0308]: mismatched types
   --> $DIR/lit.rs:16:9
diff --git a/tests/ui/rfcs/rfc-2528-type-changing-struct-update/issue-92010-trait-bound-not-satisfied.stderr b/tests/ui/rfcs/rfc-2528-type-changing-struct-update/issue-92010-trait-bound-not-satisfied.stderr
index 61aae850a94..86f2d9b6ec8 100644
--- a/tests/ui/rfcs/rfc-2528-type-changing-struct-update/issue-92010-trait-bound-not-satisfied.stderr
+++ b/tests/ui/rfcs/rfc-2528-type-changing-struct-update/issue-92010-trait-bound-not-satisfied.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |     fn y(&self, y: f64) -> Self { P{y, .. self.clone() } }
    |                                           ^^^^^^^^^^^^ expected `P<T>`, found `&P<T>`
    |
-   = note: expected struct `P<T>`
-           found reference `&P<T>`
+   = note: expected struct `P<_>`
+           found reference `&P<_>`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/static/static-reference-to-fn-1.stderr b/tests/ui/static/static-reference-to-fn-1.stderr
index ce9b6a739cf..6bf64974ef5 100644
--- a/tests/ui/static/static-reference-to-fn-1.stderr
+++ b/tests/ui/static/static-reference-to-fn-1.stderr
@@ -4,8 +4,8 @@ error[E0308]: mismatched types
 LL |         func: &foo,
    |               ^^^^ expected `&fn() -> Option<isize>`, found `&fn() -> Option<isize> {foo}`
    |
-   = note: expected reference `&fn() -> Option<isize>`
-              found reference `&fn() -> Option<isize> {foo}`
+   = note: expected reference `&fn() -> Option<_>`
+              found reference `&fn() -> Option<_> {foo}`
    = note: fn items are distinct from fn pointers
 help: consider casting to a fn pointer
    |
diff --git a/tests/ui/suggestions/as-ref.stderr b/tests/ui/suggestions/as-ref.stderr
index c5b2bb1260f..a42e2af3164 100644
--- a/tests/ui/suggestions/as-ref.stderr
+++ b/tests/ui/suggestions/as-ref.stderr
@@ -78,8 +78,8 @@ LL |     let y: Option<&usize> = x;
    |            |
    |            expected due to this
    |
-   = note:   expected enum `Option<&usize>`
-           found reference `&Option<usize>`
+   = note:   expected enum `Option<&_>`
+           found reference `&Option<_>`
 help: try using `.as_ref()` to convert `&Option<usize>` to `Option<&usize>`
    |
 LL |     let y: Option<&usize> = x.as_ref();
@@ -93,8 +93,8 @@ LL |     let y: Result<&usize, &usize> = x;
    |            |
    |            expected due to this
    |
-   = note:   expected enum `Result<&usize, &usize>`
-           found reference `&Result<usize, usize>`
+   = note:   expected enum `Result<&_, &_>`
+           found reference `&Result<_, _>`
 help: try using `.as_ref()` to convert `&Result<usize, usize>` to `Result<&usize, &usize>`
    |
 LL |     let y: Result<&usize, &usize> = x.as_ref();
@@ -108,8 +108,8 @@ LL |     let y: Result<&usize, usize> = x;
    |            |
    |            expected due to this
    |
-   = note:   expected enum `Result<&usize, usize>`
-           found reference `&Result<usize, usize>`
+   = note:   expected enum `Result<&_, _>`
+           found reference `&Result<_, _>`
 
 error[E0308]: mismatched types
   --> $DIR/as-ref.rs:22:42
diff --git a/tests/ui/suggestions/clone-on-unconstrained-borrowed-type-param.stderr b/tests/ui/suggestions/clone-on-unconstrained-borrowed-type-param.stderr
index 8c973995c34..afbb9c32d51 100644
--- a/tests/ui/suggestions/clone-on-unconstrained-borrowed-type-param.stderr
+++ b/tests/ui/suggestions/clone-on-unconstrained-borrowed-type-param.stderr
@@ -8,8 +8,8 @@ LL | fn wat<T>(t: &T) -> T {
 LL |     t.clone()
    |     ^^^^^^^^^ expected type parameter `T`, found `&T`
    |
-   = note: expected type parameter `T`
-                   found reference `&T`
+   = note: expected type parameter `_`
+                   found reference `&_`
 note: `T` does not implement `Clone`, so `&T` was cloned instead
   --> $DIR/clone-on-unconstrained-borrowed-type-param.rs:3:5
    |
diff --git a/tests/ui/suggestions/method-access-to-range-literal-typo.stderr b/tests/ui/suggestions/method-access-to-range-literal-typo.stderr
index 54a16b8efa7..b1fb0254cd9 100644
--- a/tests/ui/suggestions/method-access-to-range-literal-typo.stderr
+++ b/tests/ui/suggestions/method-access-to-range-literal-typo.stderr
@@ -18,8 +18,8 @@ LL |     fn method(&self) -> Option<&Vec<u8>> {
 LL |         self.option..as_ref().map(|x| x)
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Option<&Vec<u8>>`, found `Range<Option<Vec<u8>>>`
    |
-   = note: expected enum `Option<&Vec<u8>>`
-            found struct `std::ops::Range<Option<Vec<u8>>>`
+   = note: expected enum `Option<&Vec<_>>`
+            found struct `std::ops::Range<Option<Vec<_>>>`
 help: you likely meant to write a method call instead of a range
    |
 LL -         self.option..as_ref().map(|x| x)
diff --git a/tests/ui/suggestions/mut-ref-reassignment.stderr b/tests/ui/suggestions/mut-ref-reassignment.stderr
index b86a04c7cd3..a225b34f8c3 100644
--- a/tests/ui/suggestions/mut-ref-reassignment.stderr
+++ b/tests/ui/suggestions/mut-ref-reassignment.stderr
@@ -32,8 +32,8 @@ LL | fn suggestion2(opt: &mut Option<String>) {
 LL |     opt = Some(String::new())
    |           ^^^^^^^^^^^^^^^^^^^ expected `&mut Option<String>`, found `Option<String>`
    |
-   = note: expected mutable reference `&mut Option<String>`
-                           found enum `Option<String>`
+   = note: expected mutable reference `&mut Option<_>`
+                           found enum `Option<_>`
 help: consider dereferencing here to assign to the mutably borrowed value
    |
 LL |     *opt = Some(String::new())
diff --git a/tests/ui/traits/impl-method-mismatch.stderr b/tests/ui/traits/impl-method-mismatch.stderr
index 2655d465f23..2061fc78575 100644
--- a/tests/ui/traits/impl-method-mismatch.stderr
+++ b/tests/ui/traits/impl-method-mismatch.stderr
@@ -9,8 +9,8 @@ note: type in trait
    |
 LL |     fn jumbo(&self, x: &usize) -> usize;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   = note: expected signature `fn(&usize, &usize) -> usize`
-              found signature `unsafe fn(&usize, &usize)`
+   = note: expected signature `fn(&_, &_) -> usize`
+              found signature `unsafe fn(&_, &_)`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/traits/wrong-mul-method-signature.stderr b/tests/ui/traits/wrong-mul-method-signature.stderr
index 25a92f5ec12..91162cbc123 100644
--- a/tests/ui/traits/wrong-mul-method-signature.stderr
+++ b/tests/ui/traits/wrong-mul-method-signature.stderr
@@ -7,8 +7,8 @@ LL |     fn mul(self, s: &f64) -> Vec1 {
    |                     expected `f64`, found `&f64`
    |                     help: change the parameter type to match the trait: `f64`
    |
-   = note: expected signature `fn(Vec1, f64) -> Vec1`
-              found signature `fn(Vec1, &f64) -> Vec1`
+   = note: expected signature `fn(Vec1, _) -> Vec1`
+              found signature `fn(Vec1, &_) -> Vec1`
 
 error[E0053]: method `mul` has an incompatible type for trait
   --> $DIR/wrong-mul-method-signature.rs:33:21
diff --git a/tests/ui/type/type-check/point-at-inference-2.stderr b/tests/ui/type/type-check/point-at-inference-2.stderr
index 1d2777ad69a..8b559ffff7e 100644
--- a/tests/ui/type/type-check/point-at-inference-2.stderr
+++ b/tests/ui/type/type-check/point-at-inference-2.stderr
@@ -25,8 +25,8 @@ LL |     bar(v);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected struct `Vec<i32>`
-              found struct `Vec<&i32>`
+   = note: expected struct `Vec<_>`
+              found struct `Vec<&_>`
 note: function defined here
   --> $DIR/point-at-inference-2.rs:1:4
    |
@@ -43,8 +43,8 @@ LL |     bar(v);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected struct `Vec<i32>`
-              found struct `Vec<&i32>`
+   = note: expected struct `Vec<_>`
+              found struct `Vec<&_>`
 note: function defined here
   --> $DIR/point-at-inference-2.rs:1:4
    |
diff --git a/tests/ui/type/type-mismatch.stderr b/tests/ui/type/type-mismatch.stderr
index ce6f29d354f..aca96978ac9 100644
--- a/tests/ui/type/type-mismatch.stderr
+++ b/tests/ui/type/type-mismatch.stderr
@@ -382,8 +382,8 @@ LL |     want::<&Foo<foo>>(f);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected reference `&Foo<foo>`
-                 found struct `Foo<foo>`
+   = note: expected reference `&Foo<_>`
+                 found struct `Foo<_>`
 note: function defined here
   --> $DIR/type-mismatch.rs:14:4
    |
@@ -402,8 +402,8 @@ LL |     want::<&Foo<foo, B>>(f);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected reference `&Foo<foo, B>`
-                 found struct `Foo<foo>`
+   = note: expected reference `&Foo<_, B>`
+                 found struct `Foo<_, A>`
 note: function defined here
   --> $DIR/type-mismatch.rs:14:4
    |
@@ -546,8 +546,8 @@ LL |     want::<&Foo<foo>>(f);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected reference `&Foo<foo>`
-                 found struct `Foo<foo, B>`
+   = note: expected reference `&Foo<_, A>`
+                 found struct `Foo<_, B>`
 note: function defined here
   --> $DIR/type-mismatch.rs:14:4
    |
@@ -562,8 +562,8 @@ LL |     want::<&Foo<foo, B>>(f);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected reference `&Foo<foo, B>`
-                 found struct `Foo<foo, B>`
+   = note: expected reference `&Foo<_, _>`
+                 found struct `Foo<_, _>`
 note: function defined here
   --> $DIR/type-mismatch.rs:14:4
    |
@@ -726,8 +726,8 @@ LL |     want::<&Foo<foo>>(f);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected reference `&Foo<foo>`
-                 found struct `Foo<foo, B, A>`
+   = note: expected reference `&Foo<_, A, B>`
+                 found struct `Foo<_, B, A>`
 note: function defined here
   --> $DIR/type-mismatch.rs:14:4
    |
@@ -742,8 +742,8 @@ LL |     want::<&Foo<foo, B>>(f);
    |     |
    |     arguments to this function are incorrect
    |
-   = note: expected reference `&Foo<foo, B>`
-                 found struct `Foo<foo, B, A>`
+   = note: expected reference `&Foo<_, _, B>`
+                 found struct `Foo<_, _, A>`
 note: function defined here
   --> $DIR/type-mismatch.rs:14:4
    |
diff --git a/tests/ui/typeck/bad-index-due-to-nested.stderr b/tests/ui/typeck/bad-index-due-to-nested.stderr
index 0b705d467ff..bd7fd0392c3 100644
--- a/tests/ui/typeck/bad-index-due-to-nested.stderr
+++ b/tests/ui/typeck/bad-index-due-to-nested.stderr
@@ -44,8 +44,8 @@ LL | fn index<'a, K, V>(map: &'a HashMap<K, V>, k: K) -> &'a V {
 LL |     map[k]
    |         ^ expected `&K`, found type parameter `K`
    |
-   = note:   expected reference `&K`
-           found type parameter `K`
+   = note:   expected reference `&_`
+           found type parameter `_`
 help: consider borrowing here
    |
 LL |     map[&k]
@@ -59,8 +59,8 @@ LL | fn index<'a, K, V>(map: &'a HashMap<K, V>, k: K) -> &'a V {
 LL |     map[k]
    |     ^^^^^^ expected `&V`, found type parameter `V`
    |
-   = note:   expected reference `&'a V`
-           found type parameter `V`
+   = note:   expected reference `&'a _`
+           found type parameter `_`
 help: consider borrowing here
    |
 LL |     &map[k]
diff --git a/tests/ui/typeck/mismatched-map-under-self.stderr b/tests/ui/typeck/mismatched-map-under-self.stderr
index 41391720a28..13678b4b827 100644
--- a/tests/ui/typeck/mismatched-map-under-self.stderr
+++ b/tests/ui/typeck/mismatched-map-under-self.stderr
@@ -12,8 +12,8 @@ note: type in trait
    |
 LL |     fn values(&self) -> Self::Values;
    |               ^^^^^
-   = note: expected signature `fn(&Option<T>)`
-              found signature `fn(Option<T>)`
+   = note: expected signature `fn(&Option<_>)`
+              found signature `fn(Option<_>)`
 
 error[E0631]: type mismatch in function arguments
   --> $DIR/mismatched-map-under-self.rs:12:18
diff --git a/tests/ui/ufcs/ufcs-explicit-self-bad.rs b/tests/ui/ufcs/ufcs-explicit-self-bad.rs
index 9b0f99a189a..3bb3d906d11 100644
--- a/tests/ui/ufcs/ufcs-explicit-self-bad.rs
+++ b/tests/ui/ufcs/ufcs-explicit-self-bad.rs
@@ -39,12 +39,12 @@ impl<'a, T> SomeTrait for &'a Bar<T> {
     //~| ERROR has an incompatible type for trait
     fn dummy3(self: &&Bar<T>) {}
     //~^ ERROR mismatched `self` parameter type
-    //~| expected reference `&'a Bar<T>`
-    //~| found reference `&Bar<T>`
+    //~| expected reference `&'a Bar<_>`
+    //~| found reference `&Bar<_>`
     //~| lifetime mismatch
     //~| ERROR mismatched `self` parameter type
-    //~| expected reference `&'a Bar<T>`
-    //~| found reference `&Bar<T>`
+    //~| expected reference `&'a Bar<_>`
+    //~| found reference `&Bar<_>`
     //~| lifetime mismatch
 }
 
diff --git a/tests/ui/ufcs/ufcs-explicit-self-bad.stderr b/tests/ui/ufcs/ufcs-explicit-self-bad.stderr
index 0efaa41d48a..4c2cb0eb753 100644
--- a/tests/ui/ufcs/ufcs-explicit-self-bad.stderr
+++ b/tests/ui/ufcs/ufcs-explicit-self-bad.stderr
@@ -31,8 +31,8 @@ error[E0308]: mismatched `self` parameter type
 LL |     fn dummy2(self: &Bar<T>) {}
    |                     ^^^^^^^ lifetime mismatch
    |
-   = note: expected reference `&'a Bar<T>`
-              found reference `&Bar<T>`
+   = note: expected reference `&'a Bar<_>`
+              found reference `&Bar<_>`
 note: the anonymous lifetime defined here...
   --> $DIR/ufcs-explicit-self-bad.rs:37:21
    |
@@ -50,8 +50,8 @@ error[E0308]: mismatched `self` parameter type
 LL |     fn dummy2(self: &Bar<T>) {}
    |                     ^^^^^^^ lifetime mismatch
    |
-   = note: expected reference `&'a Bar<T>`
-              found reference `&Bar<T>`
+   = note: expected reference `&'a Bar<_>`
+              found reference `&Bar<_>`
 note: the lifetime `'a` as defined here...
   --> $DIR/ufcs-explicit-self-bad.rs:35:6
    |
@@ -69,8 +69,8 @@ error[E0308]: mismatched `self` parameter type
 LL |     fn dummy3(self: &&Bar<T>) {}
    |                     ^^^^^^^^ lifetime mismatch
    |
-   = note: expected reference `&'a Bar<T>`
-              found reference `&Bar<T>`
+   = note: expected reference `&'a Bar<_>`
+              found reference `&Bar<_>`
 note: the anonymous lifetime defined here...
   --> $DIR/ufcs-explicit-self-bad.rs:40:22
    |
@@ -88,8 +88,8 @@ error[E0308]: mismatched `self` parameter type
 LL |     fn dummy3(self: &&Bar<T>) {}
    |                     ^^^^^^^^ lifetime mismatch
    |
-   = note: expected reference `&'a Bar<T>`
-              found reference `&Bar<T>`
+   = note: expected reference `&'a Bar<_>`
+              found reference `&Bar<_>`
 note: the lifetime `'a` as defined here...
   --> $DIR/ufcs-explicit-self-bad.rs:35:6
    |
@@ -115,8 +115,8 @@ note: type in trait
    |
 LL |     fn dummy2(&self);
    |               ^^^^^
-   = note: expected signature `fn(&&'a Bar<T>)`
-              found signature `fn(&Bar<T>)`
+   = note: expected signature `fn(&&'a Bar<_>)`
+              found signature `fn(&Bar<_>)`
 
 error: aborting due to 8 previous errors
 
diff --git a/tests/ui/unsafe/unsafe-trait-impl.rs b/tests/ui/unsafe/unsafe-trait-impl.rs
index 1fc84ca0256..9fd9ff65288 100644
--- a/tests/ui/unsafe/unsafe-trait-impl.rs
+++ b/tests/ui/unsafe/unsafe-trait-impl.rs
@@ -7,8 +7,8 @@ trait Foo {
 impl Foo for u32 {
     fn len(&self) -> u32 { *self }
     //~^ ERROR method `len` has an incompatible type for trait
-    //~| expected signature `unsafe fn(&u32) -> _`
-    //~| found signature `fn(&u32) -> _`
+    //~| expected signature `unsafe fn(&_) -> _`
+    //~| found signature `fn(&_) -> _`
 }
 
 fn main() { }
diff --git a/tests/ui/unsafe/unsafe-trait-impl.stderr b/tests/ui/unsafe/unsafe-trait-impl.stderr
index db5200e1c20..5888b674d4f 100644
--- a/tests/ui/unsafe/unsafe-trait-impl.stderr
+++ b/tests/ui/unsafe/unsafe-trait-impl.stderr
@@ -9,8 +9,8 @@ note: type in trait
    |
 LL |     unsafe fn len(&self) -> u32;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   = note: expected signature `unsafe fn(&u32) -> _`
-              found signature `fn(&u32) -> _`
+   = note: expected signature `unsafe fn(&_) -> _`
+              found signature `fn(&_) -> _`
 
 error: aborting due to 1 previous error
 
-- 
cgit 1.4.1-3-g733a5


From 5fdb648fc3a672140f3fc8f98ee3268ed3fe5389 Mon Sep 17 00:00:00 2001
From: Ali MJ Al-Nasrawy <alimjalnasrawy@gmail.com>
Date: Fri, 8 Dec 2023 07:27:47 +0000
Subject: temporarily revert "ice on ambguity in mir typeck"

Reverts #116530
---
 .../src/traits/query/type_op/mod.rs                | 16 ++++-----------
 .../wf/unnormalized-projection-guides-inference.rs | 24 ++++++++++++++++++++++
 2 files changed, 28 insertions(+), 12 deletions(-)
 create mode 100644 tests/ui/wf/unnormalized-projection-guides-inference.rs

diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
index e87e585ef0b..cd09aaff2d9 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
@@ -157,18 +157,10 @@ where
         }
 
         let mut region_constraints = QueryRegionConstraints::default();
-        let (output, error_info, mut obligations) =
-            Q::fully_perform_into(self, infcx, &mut region_constraints)
-                .map_err(|_| {
-                    infcx.tcx.sess.span_delayed_bug(span, format!("error performing {self:?}"))
-                })
-                .and_then(|(output, error_info, obligations, certainty)| match certainty {
-                    Certainty::Proven => Ok((output, error_info, obligations)),
-                    Certainty::Ambiguous => Err(infcx
-                        .tcx
-                        .sess
-                        .span_delayed_bug(span, format!("ambiguity performing {self:?}"))),
-                })?;
+        let (output, error_info, mut obligations, _) =
+            Q::fully_perform_into(self, infcx, &mut region_constraints).map_err(|_| {
+                infcx.tcx.sess.span_delayed_bug(span, format!("error performing {self:?}"))
+            })?;
 
         // Typically, instantiating NLL query results does not
         // create obligations. However, in some cases there
diff --git a/tests/ui/wf/unnormalized-projection-guides-inference.rs b/tests/ui/wf/unnormalized-projection-guides-inference.rs
new file mode 100644
index 00000000000..ca2d6c2e882
--- /dev/null
+++ b/tests/ui/wf/unnormalized-projection-guides-inference.rs
@@ -0,0 +1,24 @@
+// The WF requirements of the *unnormalized* form of type annotations
+// can guide inference.
+// check-pass
+
+pub trait EqualTo {
+    type Ty;
+}
+impl<X> EqualTo for X {
+    type Ty = X;
+}
+
+trait MyTrait<U: EqualTo<Ty = Self>> {
+    type Out;
+}
+impl<T, U: EqualTo<Ty = T>> MyTrait<U> for T {
+    type Out = ();
+}
+
+fn main() {
+    let _: <_ as MyTrait<u8>>::Out;
+    // We shoud be able to infer a value for the inference variable above.
+    // The WF of the unnormalized projection requires `u8: EqualTo<Ty = _>`,
+    // which is sufficient to guide inference.
+}
-- 
cgit 1.4.1-3-g733a5


From 5d16c5423c20b3400f3286359a620378e93323d8 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 8 Dec 2023 21:02:22 +1100
Subject: Extend tidy alphabetical checking to `tests/`.

This is desired for #118702.
---
 src/tools/tidy/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs
index 80e58ba00fc..9f92b8995b7 100644
--- a/src/tools/tidy/src/main.rs
+++ b/src/tools/tidy/src/main.rs
@@ -132,6 +132,7 @@ fn main() {
         check!(edition, &library_path);
 
         check!(alphabetical, &src_path);
+        check!(alphabetical, &tests_path);
         check!(alphabetical, &compiler_path);
         check!(alphabetical, &library_path);
 
-- 
cgit 1.4.1-3-g733a5


From d2d742c4ccc8c18fe13bfc42b3a7b9b66486b294 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 23 Nov 2023 13:52:50 +1100
Subject: coverage: Add a dedicated test for coverage of `if !`

---
 tests/coverage/if_not.cov-map  | 39 +++++++++++++++++++++++++++++++++++++++
 tests/coverage/if_not.coverage | 38 ++++++++++++++++++++++++++++++++++++++
 tests/coverage/if_not.rs       | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+)
 create mode 100644 tests/coverage/if_not.cov-map
 create mode 100644 tests/coverage/if_not.coverage
 create mode 100644 tests/coverage/if_not.rs

diff --git a/tests/coverage/if_not.cov-map b/tests/coverage/if_not.cov-map
new file mode 100644
index 00000000000..73627308516
--- /dev/null
+++ b/tests/coverage/if_not.cov-map
@@ -0,0 +1,39 @@
+Function name: if_not::if_not
+Raw bytes (86): 0x[01, 01, 10, 01, 05, 05, 02, 3f, 09, 05, 02, 09, 3a, 3f, 09, 05, 02, 37, 0d, 09, 3a, 3f, 09, 05, 02, 0d, 32, 37, 0d, 09, 3a, 3f, 09, 05, 02, 0a, 01, 04, 01, 03, 0d, 02, 04, 05, 02, 06, 05, 02, 06, 00, 07, 3f, 04, 09, 00, 0d, 3a, 01, 05, 02, 06, 09, 02, 06, 00, 07, 37, 04, 09, 00, 0d, 32, 01, 05, 02, 06, 0d, 02, 0c, 02, 06, 2f, 03, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 16
+- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 2 operands: lhs = Expression(15, Add), rhs = Counter(2)
+- expression 3 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 4 operands: lhs = Counter(2), rhs = Expression(14, Sub)
+- expression 5 operands: lhs = Expression(15, Add), rhs = Counter(2)
+- expression 6 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 7 operands: lhs = Expression(13, Add), rhs = Counter(3)
+- expression 8 operands: lhs = Counter(2), rhs = Expression(14, Sub)
+- expression 9 operands: lhs = Expression(15, Add), rhs = Counter(2)
+- expression 10 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 11 operands: lhs = Counter(3), rhs = Expression(12, Sub)
+- expression 12 operands: lhs = Expression(13, Add), rhs = Counter(3)
+- expression 13 operands: lhs = Counter(2), rhs = Expression(14, Sub)
+- expression 14 operands: lhs = Expression(15, Add), rhs = Counter(2)
+- expression 15 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+Number of file 0 mappings: 10
+- Code(Counter(0)) at (prev + 4, 1) to (start + 3, 13)
+- Code(Expression(0, Sub)) at (prev + 4, 5) to (start + 2, 6)
+    = (c0 - c1)
+- Code(Counter(1)) at (prev + 2, 6) to (start + 0, 7)
+- Code(Expression(15, Add)) at (prev + 4, 9) to (start + 0, 13)
+    = (c1 + (c0 - c1))
+- Code(Expression(14, Sub)) at (prev + 1, 5) to (start + 2, 6)
+    = ((c1 + (c0 - c1)) - c2)
+- Code(Counter(2)) at (prev + 2, 6) to (start + 0, 7)
+- Code(Expression(13, Add)) at (prev + 4, 9) to (start + 0, 13)
+    = (c2 + ((c1 + (c0 - c1)) - c2))
+- Code(Expression(12, Sub)) at (prev + 1, 5) to (start + 2, 6)
+    = ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)
+- Code(Counter(3)) at (prev + 2, 12) to (start + 2, 6)
+- Code(Expression(11, Add)) at (prev + 3, 1) to (start + 0, 2)
+    = (c3 + ((c2 + ((c1 + (c0 - c1)) - c2)) - c3))
+
diff --git a/tests/coverage/if_not.coverage b/tests/coverage/if_not.coverage
new file mode 100644
index 00000000000..4c8ef9a84e2
--- /dev/null
+++ b/tests/coverage/if_not.coverage
@@ -0,0 +1,38 @@
+   LL|       |#![feature(coverage_attribute)]
+   LL|       |// edition: 2021
+   LL|       |
+   LL|     12|fn if_not(cond: bool) {
+   LL|     12|    if
+   LL|     12|        !
+   LL|     12|        cond
+   LL|      4|    {
+   LL|      4|        println!("cond was false");
+   LL|      8|    }
+   LL|       |
+   LL|       |    if
+   LL|       |        !
+   LL|     12|        cond
+   LL|      4|    {
+   LL|      4|        println!("cond was false");
+   LL|      8|    }
+   LL|       |
+   LL|       |    if
+   LL|       |        !
+   LL|     12|        cond
+   LL|      4|    {
+   LL|      4|        println!("cond was false");
+   LL|      8|    } else {
+   LL|      8|        println!("cond was true");
+   LL|      8|    }
+   LL|     12|}
+   LL|       |
+   LL|       |#[coverage(off)]
+   LL|       |fn main() {
+   LL|       |    for _ in 0..8 {
+   LL|       |        if_not(std::hint::black_box(true));
+   LL|       |    }
+   LL|       |    for _ in 0..4 {
+   LL|       |        if_not(std::hint::black_box(false));
+   LL|       |    }
+   LL|       |}
+
diff --git a/tests/coverage/if_not.rs b/tests/coverage/if_not.rs
new file mode 100644
index 00000000000..4f45ae0b3d4
--- /dev/null
+++ b/tests/coverage/if_not.rs
@@ -0,0 +1,37 @@
+#![feature(coverage_attribute)]
+// edition: 2021
+
+fn if_not(cond: bool) {
+    if
+        !
+        cond
+    {
+        println!("cond was false");
+    }
+
+    if
+        !
+        cond
+    {
+        println!("cond was false");
+    }
+
+    if
+        !
+        cond
+    {
+        println!("cond was false");
+    } else {
+        println!("cond was true");
+    }
+}
+
+#[coverage(off)]
+fn main() {
+    for _ in 0..8 {
+        if_not(std::hint::black_box(true));
+    }
+    for _ in 0..4 {
+        if_not(std::hint::black_box(false));
+    }
+}
-- 
cgit 1.4.1-3-g733a5


From 44b47aa976d6a6bdd7eb1f99a8ee5270afbe993e Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 23 Nov 2023 11:50:39 +1100
Subject: coverage: Add `CoverageKind::SpanMarker` for including extra spans in
 MIR

There are cases where coverage instrumentation wants to show a span for some
syntax element, but there is no MIR node that naturally carries that span, so
the instrumentor can't see it.

MIR building can now use this new kind of coverage statement to deliberately
include those spans in MIR, attached to a dummy statement that has no other
effect.
---
 compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs       |  3 +++
 compiler/rustc_middle/src/mir/coverage.rs                 |  8 ++++++++
 compiler/rustc_mir_build/src/build/cfg.rs                 | 13 +++++++++++++
 .../rustc_mir_transform/src/coverage/spans/from_mir.rs    | 15 ++++++++++++---
 4 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index 7d69756181a..8386f067baf 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -100,6 +100,9 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
 
         let Coverage { kind } = coverage;
         match *kind {
+            // Span markers are only meaningful during MIR instrumentation,
+            // and have no effect during codegen.
+            CoverageKind::SpanMarker => {}
             CoverageKind::CounterIncrement { id } => {
                 func_coverage.mark_counter_id_seen(id);
                 // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`,
diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs
index f15ee0082ce..ec5edceb269 100644
--- a/compiler/rustc_middle/src/mir/coverage.rs
+++ b/compiler/rustc_middle/src/mir/coverage.rs
@@ -76,6 +76,13 @@ impl Debug for CovTerm {
 
 #[derive(Clone, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub enum CoverageKind {
+    /// Marks a span that might otherwise not be represented in MIR, so that
+    /// coverage instrumentation can associate it with its enclosing block/BCB.
+    ///
+    /// Only used by the `InstrumentCoverage` pass, and has no effect during
+    /// codegen.
+    SpanMarker,
+
     /// Marks the point in MIR control flow represented by a coverage counter.
     ///
     /// This is eventually lowered to `llvm.instrprof.increment` in LLVM IR.
@@ -99,6 +106,7 @@ impl Debug for CoverageKind {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         use CoverageKind::*;
         match self {
+            SpanMarker => write!(fmt, "SpanMarker"),
             CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()),
             ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()),
         }
diff --git a/compiler/rustc_mir_build/src/build/cfg.rs b/compiler/rustc_mir_build/src/build/cfg.rs
index fddcf9de7c7..2bd0e289731 100644
--- a/compiler/rustc_mir_build/src/build/cfg.rs
+++ b/compiler/rustc_mir_build/src/build/cfg.rs
@@ -101,6 +101,19 @@ impl<'tcx> CFG<'tcx> {
         self.push(block, stmt);
     }
 
+    /// Adds a dummy statement whose only role is to associate a span with its
+    /// enclosing block for the purposes of coverage instrumentation.
+    ///
+    /// This results in more accurate coverage reports for certain kinds of
+    /// syntax (e.g. `continue` or `if !`) that would otherwise not appear in MIR.
+    pub(crate) fn push_coverage_span_marker(&mut self, block: BasicBlock, source_info: SourceInfo) {
+        let kind = StatementKind::Coverage(Box::new(Coverage {
+            kind: coverage::CoverageKind::SpanMarker,
+        }));
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
     pub(crate) fn terminate(
         &mut self,
         block: BasicBlock,
diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
index e1531f2c239..6f7d8d9dd75 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
@@ -92,13 +92,13 @@ fn is_closure(statement: &Statement<'_>) -> bool {
 /// If the MIR `Statement` has a span contributive to computing coverage spans,
 /// return it; otherwise return `None`.
 fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
+    use mir::coverage::CoverageKind;
+
     match statement.kind {
         // These statements have spans that are often outside the scope of the executed source code
         // for their parent `BasicBlock`.
         StatementKind::StorageLive(_)
         | StatementKind::StorageDead(_)
-        // Coverage should not be encountered, but don't inject coverage coverage
-        | StatementKind::Coverage(_)
         // Ignore `ConstEvalCounter`s
         | StatementKind::ConstEvalCounter
         // Ignore `Nop`s
@@ -122,9 +122,13 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
         // If and when the Issue is resolved, remove this special case match pattern:
         StatementKind::FakeRead(box (FakeReadCause::ForGuardBinding, _)) => None,
 
-        // Retain spans from all other statements
+        // Retain spans from most other statements.
         StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding`
         | StatementKind::Intrinsic(..)
+        | StatementKind::Coverage(box mir::Coverage {
+            // The purpose of `SpanMarker` is to be matched and accepted here.
+            kind: CoverageKind::SpanMarker
+        })
         | StatementKind::Assign(_)
         | StatementKind::SetDiscriminant { .. }
         | StatementKind::Deinit(..)
@@ -133,6 +137,11 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
         | StatementKind::AscribeUserType(_, _) => {
             Some(statement.source_info.span)
         }
+
+        StatementKind::Coverage(box mir::Coverage {
+            // These coverage statements should not exist prior to coverage instrumentation.
+            kind: CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. }
+        }) => bug!("Unexpected coverage statement found during coverage instrumentation: {statement:?}"),
     }
 }
 
-- 
cgit 1.4.1-3-g733a5


From 98166358a99aea4ded1a42a3c113aac764c752ed Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 23 Nov 2023 11:59:13 +1100
Subject: coverage: Use `SpanMarker` to mark `continue` expressions.

This replaces the previous workaround, which was to inject a dummy `Assign`
statement.
---
 compiler/rustc_mir_build/src/build/scope.rs | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index 993fee95895..88fcaa0a41c 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -90,7 +90,6 @@ use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
 use rustc_middle::thir::{Expr, LintLevel};
-use rustc_middle::ty::Ty;
 use rustc_session::lint::Level;
 use rustc_span::{Span, DUMMY_SP};
 
@@ -660,14 +659,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             (None, Some(_)) => {
                 panic!("`return`, `become` and `break` with value and must have a destination")
             }
-            (None, None) if self.tcx.sess.instrument_coverage() => {
-                // Unlike `break` and `return`, which push an `Assign` statement to MIR, from which
-                // a Coverage code region can be generated, `continue` needs no `Assign`; but
-                // without one, the `InstrumentCoverage` MIR pass cannot generate a code region for
-                // `continue`. Coverage will be missing unless we add a dummy `Assign` to MIR.
-                self.add_dummy_assignment(span, block, source_info);
+            (None, None) => {
+                if self.tcx.sess.instrument_coverage() {
+                    // Normally we wouldn't build any MIR in this case, but that makes it
+                    // harder for coverage instrumentation to extract a relevant span for
+                    // `continue` expressions. So here we inject a dummy statement with the
+                    // desired span.
+                    self.cfg.push_coverage_span_marker(block, source_info);
+                }
             }
-            (None, None) => {}
         }
 
         let region_scope = self.scopes.breakable_scopes[break_index].region_scope;
@@ -723,14 +723,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         self.cfg.terminate(block, source_info, TerminatorKind::UnwindResume);
     }
 
-    // Add a dummy `Assign` statement to the CFG, with the span for the source code's `continue`
-    // statement.
-    fn add_dummy_assignment(&mut self, span: Span, block: BasicBlock, source_info: SourceInfo) {
-        let local_decl = LocalDecl::new(Ty::new_unit(self.tcx), span);
-        let temp_place = Place::from(self.local_decls.push(local_decl));
-        self.cfg.push_assign_unit(block, source_info, temp_place, self.tcx);
-    }
-
     fn leave_top_scope(&mut self, block: BasicBlock) -> BasicBlock {
         // If we are emitting a `drop` statement, we need to have the cached
         // diverge cleanup pads ready in case that drop panics.
-- 
cgit 1.4.1-3-g733a5


From d90fd027c887247caa501db5f49352fdfab769d9 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Thu, 23 Nov 2023 12:50:40 +1100
Subject: coverage: Use `SpanMarker` to mark the full condition of `if !`

When MIR is built for an if-not expression, the `!` part of the condition
doesn't correspond to any MIR statement, so coverage instrumentation normally
can't see it.

We can fix that by deliberately injecting a dummy statement whose sole purpose
is to associate that span with its enclosing block.
---
 compiler/rustc_mir_build/src/build/matches/mod.rs |  5 +++++
 tests/coverage/if_not.cov-map                     | 10 +++++-----
 tests/coverage/if_not.coverage                    |  4 ++--
 tests/coverage/lazy_boolean.cov-map               |  8 ++++----
 tests/coverage/lazy_boolean.coverage              |  2 +-
 5 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index 90f950d59d5..541b87af797 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -90,6 +90,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let local_scope = this.local_scope();
                 let (success_block, failure_block) =
                     this.in_if_then_scope(local_scope, expr_span, |this| {
+                        // Help out coverage instrumentation by injecting a dummy statement with
+                        // the original condition's span (including `!`). This fixes #115468.
+                        if this.tcx.sess.instrument_coverage() {
+                            this.cfg.push_coverage_span_marker(block, this.source_info(expr_span));
+                        }
                         this.then_else_break(
                             block,
                             &this.thir[arg],
diff --git a/tests/coverage/if_not.cov-map b/tests/coverage/if_not.cov-map
index 73627308516..fb893e37960 100644
--- a/tests/coverage/if_not.cov-map
+++ b/tests/coverage/if_not.cov-map
@@ -1,5 +1,5 @@
 Function name: if_not::if_not
-Raw bytes (86): 0x[01, 01, 10, 01, 05, 05, 02, 3f, 09, 05, 02, 09, 3a, 3f, 09, 05, 02, 37, 0d, 09, 3a, 3f, 09, 05, 02, 0d, 32, 37, 0d, 09, 3a, 3f, 09, 05, 02, 0a, 01, 04, 01, 03, 0d, 02, 04, 05, 02, 06, 05, 02, 06, 00, 07, 3f, 04, 09, 00, 0d, 3a, 01, 05, 02, 06, 09, 02, 06, 00, 07, 37, 04, 09, 00, 0d, 32, 01, 05, 02, 06, 0d, 02, 0c, 02, 06, 2f, 03, 01, 00, 02]
+Raw bytes (86): 0x[01, 01, 10, 01, 05, 05, 02, 3f, 09, 05, 02, 09, 3a, 3f, 09, 05, 02, 37, 0d, 09, 3a, 3f, 09, 05, 02, 0d, 32, 37, 0d, 09, 3a, 3f, 09, 05, 02, 0a, 01, 04, 01, 03, 0d, 02, 04, 05, 02, 06, 05, 02, 06, 00, 07, 3f, 03, 09, 01, 0d, 3a, 02, 05, 02, 06, 09, 02, 06, 00, 07, 37, 03, 09, 01, 0d, 32, 02, 05, 02, 06, 0d, 02, 0c, 02, 06, 2f, 03, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 16
@@ -24,14 +24,14 @@ Number of file 0 mappings: 10
 - Code(Expression(0, Sub)) at (prev + 4, 5) to (start + 2, 6)
     = (c0 - c1)
 - Code(Counter(1)) at (prev + 2, 6) to (start + 0, 7)
-- Code(Expression(15, Add)) at (prev + 4, 9) to (start + 0, 13)
+- Code(Expression(15, Add)) at (prev + 3, 9) to (start + 1, 13)
     = (c1 + (c0 - c1))
-- Code(Expression(14, Sub)) at (prev + 1, 5) to (start + 2, 6)
+- Code(Expression(14, Sub)) at (prev + 2, 5) to (start + 2, 6)
     = ((c1 + (c0 - c1)) - c2)
 - Code(Counter(2)) at (prev + 2, 6) to (start + 0, 7)
-- Code(Expression(13, Add)) at (prev + 4, 9) to (start + 0, 13)
+- Code(Expression(13, Add)) at (prev + 3, 9) to (start + 1, 13)
     = (c2 + ((c1 + (c0 - c1)) - c2))
-- Code(Expression(12, Sub)) at (prev + 1, 5) to (start + 2, 6)
+- Code(Expression(12, Sub)) at (prev + 2, 5) to (start + 2, 6)
     = ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)
 - Code(Counter(3)) at (prev + 2, 12) to (start + 2, 6)
 - Code(Expression(11, Add)) at (prev + 3, 1) to (start + 0, 2)
diff --git a/tests/coverage/if_not.coverage b/tests/coverage/if_not.coverage
index 4c8ef9a84e2..41838b8513f 100644
--- a/tests/coverage/if_not.coverage
+++ b/tests/coverage/if_not.coverage
@@ -10,14 +10,14 @@
    LL|      8|    }
    LL|       |
    LL|       |    if
-   LL|       |        !
+   LL|     12|        !
    LL|     12|        cond
    LL|      4|    {
    LL|      4|        println!("cond was false");
    LL|      8|    }
    LL|       |
    LL|       |    if
-   LL|       |        !
+   LL|     12|        !
    LL|     12|        cond
    LL|      4|    {
    LL|      4|        println!("cond was false");
diff --git a/tests/coverage/lazy_boolean.cov-map b/tests/coverage/lazy_boolean.cov-map
index 0ad393c40fa..2d1ff24e62d 100644
--- a/tests/coverage/lazy_boolean.cov-map
+++ b/tests/coverage/lazy_boolean.cov-map
@@ -1,5 +1,5 @@
 Function name: lazy_boolean::main
-Raw bytes (636): 0x[01, 01, a4, 01, 01, 05, 09, 8a, 05, 8f, 05, 09, 05, 02, 05, 02, 8f, 05, 09, 05, 02, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 09, 8a, 05, 8f, 05, 09, 05, 02, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 21, da, 04, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, d7, 04, 25, 21, da, 04, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 25, d2, 04, d7, 04, 25, 21, da, 04, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 1c, 01, 03, 01, 07, 0f, 05, 07, 10, 04, 06, 02, 04, 06, 00, 07, 87, 05, 02, 09, 00, 11, 8f, 05, 02, 0d, 00, 12, 8a, 05, 02, 0d, 00, 12, ff, 04, 03, 09, 00, 11, 87, 05, 02, 0d, 00, 12, 82, 05, 02, 0d, 00, 12, f7, 04, 02, 09, 00, 11, ff, 04, 00, 14, 00, 19, 11, 00, 1d, 00, 22, ef, 04, 01, 09, 00, 11, f7, 04, 00, 14, 00, 19, 15, 00, 1d, 00, 22, ef, 04, 04, 09, 00, 10, ea, 04, 01, 05, 03, 06, 19, 03, 06, 00, 07, e7, 04, 03, 09, 00, 10, 1d, 01, 05, 03, 06, e2, 04, 05, 05, 03, 06, df, 04, 05, 09, 00, 10, da, 04, 00, 11, 02, 06, 21, 02, 06, 00, 07, d7, 04, 02, 08, 00, 0f, 25, 00, 10, 02, 06, d2, 04, 02, 0c, 02, 06, cf, 04, 03, 01, 00, 02]
+Raw bytes (636): 0x[01, 01, a4, 01, 01, 05, 09, 8a, 05, 8f, 05, 09, 05, 02, 05, 02, 8f, 05, 09, 05, 02, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 09, 8a, 05, 8f, 05, 09, 05, 02, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 21, da, 04, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, d7, 04, 25, 21, da, 04, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 25, d2, 04, d7, 04, 25, 21, da, 04, df, 04, 21, 1d, e2, 04, e7, 04, 1d, 19, ea, 04, ef, 04, 19, 15, f2, 04, f7, 04, 15, 11, fa, 04, ff, 04, 11, 0d, 82, 05, 87, 05, 0d, 09, 8a, 05, 8f, 05, 09, 05, 02, 1c, 01, 03, 01, 07, 0f, 05, 07, 10, 04, 06, 02, 04, 06, 00, 07, 87, 05, 02, 09, 00, 11, 8f, 05, 02, 0d, 00, 12, 8a, 05, 02, 0d, 00, 12, ff, 04, 03, 09, 00, 11, 87, 05, 02, 0d, 00, 12, 82, 05, 02, 0d, 00, 12, f7, 04, 02, 09, 00, 11, ff, 04, 00, 14, 00, 19, 11, 00, 1d, 00, 22, ef, 04, 01, 09, 00, 11, f7, 04, 00, 14, 00, 19, 15, 00, 1d, 00, 22, ef, 04, 03, 09, 01, 10, ea, 04, 02, 05, 03, 06, 19, 03, 06, 00, 07, e7, 04, 03, 09, 00, 10, 1d, 01, 05, 03, 06, e2, 04, 05, 05, 03, 06, df, 04, 05, 08, 00, 10, da, 04, 00, 11, 02, 06, 21, 02, 06, 00, 07, d7, 04, 02, 08, 00, 0f, 25, 00, 10, 02, 06, d2, 04, 02, 0c, 02, 06, cf, 04, 03, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 164
@@ -194,9 +194,9 @@ Number of file 0 mappings: 28
 - Code(Expression(157, Add)) at (prev + 0, 20) to (start + 0, 25)
     = (c4 + ((c3 + ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)) - c4))
 - Code(Counter(5)) at (prev + 0, 29) to (start + 0, 34)
-- Code(Expression(155, Add)) at (prev + 4, 9) to (start + 0, 16)
+- Code(Expression(155, Add)) at (prev + 3, 9) to (start + 1, 16)
     = (c5 + ((c4 + ((c3 + ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)) - c4)) - c5))
-- Code(Expression(154, Sub)) at (prev + 1, 5) to (start + 3, 6)
+- Code(Expression(154, Sub)) at (prev + 2, 5) to (start + 3, 6)
     = ((c5 + ((c4 + ((c3 + ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)) - c4)) - c5)) - c6)
 - Code(Counter(6)) at (prev + 3, 6) to (start + 0, 7)
 - Code(Expression(153, Add)) at (prev + 3, 9) to (start + 0, 16)
@@ -204,7 +204,7 @@ Number of file 0 mappings: 28
 - Code(Counter(7)) at (prev + 1, 5) to (start + 3, 6)
 - Code(Expression(152, Sub)) at (prev + 5, 5) to (start + 3, 6)
     = ((c6 + ((c5 + ((c4 + ((c3 + ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)) - c4)) - c5)) - c6)) - c7)
-- Code(Expression(151, Add)) at (prev + 5, 9) to (start + 0, 16)
+- Code(Expression(151, Add)) at (prev + 5, 8) to (start + 0, 16)
     = (c7 + ((c6 + ((c5 + ((c4 + ((c3 + ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)) - c4)) - c5)) - c6)) - c7))
 - Code(Expression(150, Sub)) at (prev + 0, 17) to (start + 2, 6)
     = ((c7 + ((c6 + ((c5 + ((c4 + ((c3 + ((c2 + ((c1 + (c0 - c1)) - c2)) - c3)) - c4)) - c5)) - c6)) - c7)) - c8)
diff --git a/tests/coverage/lazy_boolean.coverage b/tests/coverage/lazy_boolean.coverage
index 8f14082ef68..2d927a08356 100644
--- a/tests/coverage/lazy_boolean.coverage
+++ b/tests/coverage/lazy_boolean.coverage
@@ -32,7 +32,7 @@
                                           ^0
    LL|       |
    LL|       |    if
-   LL|       |        !
+   LL|      1|        !
    LL|      1|        is_true
    LL|      0|    {
    LL|      0|        a = 2
-- 
cgit 1.4.1-3-g733a5


From e0cd8057c85260e827e417cfcf3c6c861d2c8426 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Wed, 6 Dec 2023 18:40:06 +1100
Subject: coverage: Simplify the heuristic for ignoring `async fn` return spans

---
 compiler/rustc_mir_transform/src/coverage/spans.rs  | 21 ++++-----------------
 .../src/coverage/spans/from_mir.rs                  | 10 ++++++++++
 2 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index c415a832994..0124bb7337c 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -319,29 +319,16 @@ impl<'a> CoverageSpansGenerator<'a> {
             }
         }
 
-        let prev = self.take_prev();
-        debug!("    AT END, adding last prev={prev:?}");
-
         // Drain any remaining dups into the output.
         for dup in self.pending_dups.drain(..) {
             debug!("    ...adding at least one pending dup={:?}", dup);
             self.refined_spans.push(dup);
         }
 
-        // Async functions wrap a closure that implements the body to be executed. The enclosing
-        // function is called and returns an `impl Future` without initially executing any of the
-        // body. To avoid showing the return from the enclosing function as a "covered" return from
-        // the closure, the enclosing function's `TerminatorKind::Return`s `CoverageSpan` is
-        // excluded. The closure's `Return` is the only one that will be counted. This provides
-        // adequate coverage, and more intuitive counts. (Avoids double-counting the closing brace
-        // of the function body.)
-        let body_ends_with_closure = if let Some(last_covspan) = self.refined_spans.last() {
-            last_covspan.is_closure && last_covspan.span.hi() == self.body_span.hi()
-        } else {
-            false
-        };
-
-        if !body_ends_with_closure {
+        // There is usually a final span remaining in `prev` after the loop ends,
+        // so add it to the output as well.
+        if let Some(prev) = self.some_prev.take() {
+            debug!("    AT END, adding last prev={prev:?}");
             self.refined_spans.push(prev);
         }
 
diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
index e1531f2c239..b850b3374ba 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
@@ -44,6 +44,16 @@ pub(super) fn mir_to_initial_sorted_coverage_spans(
             .then_with(|| Ord::cmp(&a.is_closure, &b.is_closure).reverse())
     });
 
+    // The desugaring of an async function includes a closure containing the
+    // original function body, and a terminator that returns the `impl Future`.
+    // That terminator will cause a confusing coverage count for the function's
+    // closing brace, so discard everything after the body closure span.
+    if let Some(body_closure_index) =
+        initial_spans.iter().rposition(|covspan| covspan.is_closure && covspan.span == body_span)
+    {
+        initial_spans.truncate(body_closure_index + 1);
+    }
+
     initial_spans
 }
 
-- 
cgit 1.4.1-3-g733a5


From cec814202a7941f92ed0f3e54d66d95b1ebcd74e Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Wed, 6 Dec 2023 21:07:43 +1100
Subject: coverage: Add `#[track_caller]` to the span generator's unwrap
 methods

This should make it easier to investigate unwrap failures in bug reports.
---
 compiler/rustc_mir_transform/src/coverage/spans.rs | 26 ++++++++++------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index 0124bb7337c..05ad14f1525 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -385,38 +385,36 @@ impl<'a> CoverageSpansGenerator<'a> {
         self.refined_spans.push(macro_name_cov);
     }
 
+    #[track_caller]
     fn curr(&self) -> &CoverageSpan {
-        self.some_curr
-            .as_ref()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_curr"))
+        self.some_curr.as_ref().unwrap_or_else(|| bug!("some_curr is None (curr)"))
     }
 
+    #[track_caller]
     fn curr_mut(&mut self) -> &mut CoverageSpan {
-        self.some_curr
-            .as_mut()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_curr"))
+        self.some_curr.as_mut().unwrap_or_else(|| bug!("some_curr is None (curr_mut)"))
     }
 
     /// If called, then the next call to `next_coverage_span()` will *not* update `prev` with the
     /// `curr` coverage span.
+    #[track_caller]
     fn take_curr(&mut self) -> CoverageSpan {
-        self.some_curr.take().unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_curr"))
+        self.some_curr.take().unwrap_or_else(|| bug!("some_curr is None (take_curr)"))
     }
 
+    #[track_caller]
     fn prev(&self) -> &CoverageSpan {
-        self.some_prev
-            .as_ref()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_prev"))
+        self.some_prev.as_ref().unwrap_or_else(|| bug!("some_prev is None (prev)"))
     }
 
+    #[track_caller]
     fn prev_mut(&mut self) -> &mut CoverageSpan {
-        self.some_prev
-            .as_mut()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_prev"))
+        self.some_prev.as_mut().unwrap_or_else(|| bug!("some_prev is None (prev_mut)"))
     }
 
+    #[track_caller]
     fn take_prev(&mut self) -> CoverageSpan {
-        self.some_prev.take().unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_prev"))
+        self.some_prev.take().unwrap_or_else(|| bug!("some_prev is None (take_prev)"))
     }
 
     /// If there are `pending_dups` but `prev` is not a matching dup (`prev.span` doesn't match the
-- 
cgit 1.4.1-3-g733a5


From e01338aeb8d83f373ff2d563b147456a68c751e5 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Fri, 8 Dec 2023 15:51:06 +1100
Subject: coverage: Regression test for unwrapping `prev` when there are no
 spans

---
 tests/coverage/no_spans.cov-map  |  8 ++++++++
 tests/coverage/no_spans.coverage | 30 ++++++++++++++++++++++++++++++
 tests/coverage/no_spans.rs       | 29 +++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 tests/coverage/no_spans.cov-map
 create mode 100644 tests/coverage/no_spans.coverage
 create mode 100644 tests/coverage/no_spans.rs

diff --git a/tests/coverage/no_spans.cov-map b/tests/coverage/no_spans.cov-map
new file mode 100644
index 00000000000..9915fc52e6d
--- /dev/null
+++ b/tests/coverage/no_spans.cov-map
@@ -0,0 +1,8 @@
+Function name: no_spans::affected_function::{closure#0}
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 1b, 0c, 00, 0e]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 27, 12) to (start + 0, 14)
+
diff --git a/tests/coverage/no_spans.coverage b/tests/coverage/no_spans.coverage
new file mode 100644
index 00000000000..e55177698a2
--- /dev/null
+++ b/tests/coverage/no_spans.coverage
@@ -0,0 +1,30 @@
+   LL|       |#![feature(coverage_attribute)]
+   LL|       |// edition: 2021
+   LL|       |
+   LL|       |// If the span extractor can't find any relevant spans for a function, the
+   LL|       |// refinement loop will terminate with nothing in its `prev` slot. If the
+   LL|       |// subsequent code tries to unwrap `prev`, it will panic.
+   LL|       |//
+   LL|       |// This scenario became more likely after #118525 started discarding spans that
+   LL|       |// can't be un-expanded back to within the function body.
+   LL|       |//
+   LL|       |// Regression test for "invalid attempt to unwrap a None some_prev", as seen
+   LL|       |// in issues such as #118643 and #118662.
+   LL|       |
+   LL|       |#[coverage(off)]
+   LL|       |fn main() {
+   LL|       |    affected_function()();
+   LL|       |}
+   LL|       |
+   LL|       |macro_rules! macro_that_defines_a_function {
+   LL|       |    (fn $name:ident () $body:tt) => {
+   LL|       |        fn $name () -> impl Fn() $body
+   LL|       |    }
+   LL|       |}
+   LL|       |
+   LL|       |macro_that_defines_a_function! {
+   LL|       |    fn affected_function() {
+   LL|      1|        || ()
+   LL|       |    }
+   LL|       |}
+
diff --git a/tests/coverage/no_spans.rs b/tests/coverage/no_spans.rs
new file mode 100644
index 00000000000..a5234bc6b60
--- /dev/null
+++ b/tests/coverage/no_spans.rs
@@ -0,0 +1,29 @@
+#![feature(coverage_attribute)]
+// edition: 2021
+
+// If the span extractor can't find any relevant spans for a function, the
+// refinement loop will terminate with nothing in its `prev` slot. If the
+// subsequent code tries to unwrap `prev`, it will panic.
+//
+// This scenario became more likely after #118525 started discarding spans that
+// can't be un-expanded back to within the function body.
+//
+// Regression test for "invalid attempt to unwrap a None some_prev", as seen
+// in issues such as #118643 and #118662.
+
+#[coverage(off)]
+fn main() {
+    affected_function()();
+}
+
+macro_rules! macro_that_defines_a_function {
+    (fn $name:ident () $body:tt) => {
+        fn $name () -> impl Fn() $body
+    }
+}
+
+macro_that_defines_a_function! {
+    fn affected_function() {
+        || ()
+    }
+}
-- 
cgit 1.4.1-3-g733a5


From b378059e6b2573c5356423fa31d184a89a3b6029 Mon Sep 17 00:00:00 2001
From: Krasimir Georgiev <krasimir@google.com>
Date: Mon, 4 Dec 2023 15:52:26 +0000
Subject: update target feature following LLVM API change

LLVM commit https://github.com/llvm/llvm-project/commit/e81796671890b59c110f8e41adc7ca26f8484d20
renamed the `unaligned-scalar-mem` target feature to `fast-unaligned-access`.
---
 compiler/rustc_codegen_llvm/src/llvm_util.rs      | 4 ++++
 compiler/rustc_codegen_ssa/src/target_features.rs | 2 +-
 tests/ui/abi/riscv-discoverability-guidance.rs    | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index eb69efb0d59..93cb7327a01 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -263,6 +263,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> {
             "sve2-bitperm",
             TargetFeatureFoldStrength::EnableOnly("neon"),
         ),
+        // The unaligned-scalar-mem feature was renamed to fast-unaligned-access.
+        ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => {
+            LLVMFeature::new("unaligned-scalar-mem")
+        }
         (_, s) => LLVMFeature::new(s),
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index d802816bb75..c3b8859c779 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -291,9 +291,9 @@ const RISCV_ALLOWED_FEATURES: &[(&str, Stability)] = &[
     ("d", Unstable(sym::riscv_target_feature)),
     ("e", Unstable(sym::riscv_target_feature)),
     ("f", Unstable(sym::riscv_target_feature)),
+    ("fast-unaligned-access", Unstable(sym::riscv_target_feature)),
     ("m", Stable),
     ("relax", Unstable(sym::riscv_target_feature)),
-    ("unaligned-scalar-mem", Unstable(sym::riscv_target_feature)),
     ("v", Unstable(sym::riscv_target_feature)),
     ("zba", Stable),
     ("zbb", Stable),
diff --git a/tests/ui/abi/riscv-discoverability-guidance.rs b/tests/ui/abi/riscv-discoverability-guidance.rs
index f57fcd6044f..361ed8f3d91 100644
--- a/tests/ui/abi/riscv-discoverability-guidance.rs
+++ b/tests/ui/abi/riscv-discoverability-guidance.rs
@@ -2,9 +2,9 @@
 // revisions: riscv32 riscv64
 //
 // [riscv32] needs-llvm-components: riscv
-// [riscv32] compile-flags: --target=riscv32i-unknown-none-elf -C target-feature=-unaligned-scalar-mem --crate-type=rlib
+// [riscv32] compile-flags: --target=riscv32i-unknown-none-elf -C target-feature=-fast-unaligned-access --crate-type=rlib
 // [riscv64] needs-llvm-components: riscv
-// [riscv64] compile-flags: --target=riscv64gc-unknown-none-elf -C target-feature=-unaligned-scalar-mem --crate-type=rlib
+// [riscv64] compile-flags: --target=riscv64gc-unknown-none-elf -C target-feature=-fast-unaligned-access --crate-type=rlib
 #![no_core]
 #![feature(
     no_core,
-- 
cgit 1.4.1-3-g733a5


From 5d977240023003f9f8d82e42fd7d45bbfd630954 Mon Sep 17 00:00:00 2001
From: David Wood <david.wood@huawei.com>
Date: Thu, 7 Dec 2023 17:06:08 +0000
Subject: privacy: visit trait def id of projections

A refactoring in #117076 changed the `DefIdVisitorSkeleton` to avoid
calling `visit_projection_ty` for `ty::Projection` aliases, and instead
just iterate over the args - this makes sense, as `visit_projection_ty`
will indirectly visit all of the same args, but in doing so, will also
create a `TraitRef` containing the trait's `DefId`, which also gets
visited. The trait's `DefId` isn't visited when we only visit the
arguments without separating them into `TraitRef` and own args first.

Signed-off-by: David Wood <david@davidtw.co>
---
 compiler/rustc_privacy/src/lib.rs          | 13 ++++----
 tests/ui/privacy/auxiliary/issue-117997.rs | 35 +++++++++++++++++++++
 tests/ui/privacy/issue-117997.rs           |  8 +++++
 tests/ui/privacy/private-in-public.rs      |  2 ++
 tests/ui/privacy/private-in-public.stderr  | 50 ++++++++++++++++++++++--------
 5 files changed, 89 insertions(+), 19 deletions(-)
 create mode 100644 tests/ui/privacy/auxiliary/issue-117997.rs
 create mode 100644 tests/ui/privacy/issue-117997.rs

diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs
index 9064cb6e875..e49282e638a 100644
--- a/compiler/rustc_privacy/src/lib.rs
+++ b/compiler/rustc_privacy/src/lib.rs
@@ -218,20 +218,21 @@ where
                     return ControlFlow::Continue(());
                 }
 
-                let kind = match kind {
-                    ty::Inherent | ty::Projection => "associated type",
-                    ty::Weak => "type alias",
-                    ty::Opaque => unreachable!(),
-                };
                 self.def_id_visitor.visit_def_id(
                     data.def_id,
-                    kind,
+                    match kind {
+                        ty::Inherent | ty::Projection => "associated type",
+                        ty::Weak => "type alias",
+                        ty::Opaque => unreachable!(),
+                    },
                     &LazyDefPathStr { def_id: data.def_id, tcx },
                 )?;
 
                 // This will also visit args if necessary, so we don't need to recurse.
                 return if V::SHALLOW {
                     ControlFlow::Continue(())
+                } else if kind == ty::Projection {
+                    self.visit_projection_ty(data)
                 } else {
                     data.args.iter().try_for_each(|subst| subst.visit_with(self))
                 };
diff --git a/tests/ui/privacy/auxiliary/issue-117997.rs b/tests/ui/privacy/auxiliary/issue-117997.rs
new file mode 100644
index 00000000000..6f71cc2ba35
--- /dev/null
+++ b/tests/ui/privacy/auxiliary/issue-117997.rs
@@ -0,0 +1,35 @@
+// no-prefer-dynamic
+// compile-flags: --crate-type=rlib
+
+pub use impl_mod::TraitImplementer as Implementer;
+
+pub use trait_mod::get_assoc;
+
+mod impl_mod {
+    use crate::trait_mod::TraitWithAssocType;
+
+    pub struct TraitImplementer {}
+    pub struct AssociatedType {}
+
+    impl AssociatedType {
+        pub fn method_on_assoc(&self) -> i32 {
+            todo!()
+        }
+    }
+
+    impl TraitWithAssocType for TraitImplementer {
+        type AssocType = AssociatedType;
+    }
+}
+
+mod trait_mod {
+    use crate::Implementer;
+
+    pub fn get_assoc() -> <Implementer as TraitWithAssocType>::AssocType {
+        todo!()
+    }
+
+    pub trait TraitWithAssocType {
+        type AssocType;
+    }
+}
diff --git a/tests/ui/privacy/issue-117997.rs b/tests/ui/privacy/issue-117997.rs
new file mode 100644
index 00000000000..d8284ef2997
--- /dev/null
+++ b/tests/ui/privacy/issue-117997.rs
@@ -0,0 +1,8 @@
+// aux-build:issue-117997.rs
+// build-pass
+
+extern crate issue_117997;
+
+pub fn main() {
+    issue_117997::get_assoc().method_on_assoc();
+}
diff --git a/tests/ui/privacy/private-in-public.rs b/tests/ui/privacy/private-in-public.rs
index 3fff2d51710..7b8e0fbe6b6 100644
--- a/tests/ui/privacy/private-in-public.rs
+++ b/tests/ui/privacy/private-in-public.rs
@@ -106,6 +106,7 @@ mod aliases_pub {
     pub fn f3(arg: <Priv as PrivTr>::Assoc) {}
     //~^ WARNING type `aliases_pub::Priv` is more private than the item `aliases_pub::f3`
     //~| WARNING associated type `aliases_pub::PrivTr::Assoc` is more private than the item `aliases_pub::f3`
+    //~^^^ WARNING trait `aliases_pub::PrivTr` is more private than the item `aliases_pub::f3`
 
     impl PrivUseAlias {
         pub fn f(arg: Priv) {}
@@ -135,6 +136,7 @@ mod aliases_priv {
     pub fn f3(arg: <Priv as PrivTr>::Assoc) {}
     //~^ WARNING type `aliases_priv::Priv` is more private than the item `aliases_priv::f3`
     //~| WARNING associated type `aliases_priv::PrivTr::Assoc` is more private than the item `aliases_priv::f3`
+    //~^^^ WARNING trait `aliases_priv::PrivTr` is more private than the item `aliases_priv::f3`
 }
 
 mod aliases_params {
diff --git a/tests/ui/privacy/private-in-public.stderr b/tests/ui/privacy/private-in-public.stderr
index 49cc2e19bf0..ff3061337ff 100644
--- a/tests/ui/privacy/private-in-public.stderr
+++ b/tests/ui/privacy/private-in-public.stderr
@@ -288,6 +288,18 @@ note: but associated type `aliases_pub::PrivTr::Assoc` is only usable at visibil
 LL |         type Assoc = m::Pub3;
    |         ^^^^^^^^^^
 
+warning: trait `aliases_pub::PrivTr` is more private than the item `aliases_pub::f3`
+  --> $DIR/private-in-public.rs:106:5
+   |
+LL |     pub fn f3(arg: <Priv as PrivTr>::Assoc) {}
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_pub::f3` is reachable at visibility `pub(crate)`
+   |
+note: but trait `aliases_pub::PrivTr` is only usable at visibility `pub(self)`
+  --> $DIR/private-in-public.rs:100:5
+   |
+LL |     trait PrivTr {
+   |     ^^^^^^^^^^^^
+
 warning: type `aliases_pub::Priv` is more private than the item `aliases_pub::f3`
   --> $DIR/private-in-public.rs:106:5
    |
@@ -301,76 +313,88 @@ LL |     struct Priv;
    |     ^^^^^^^^^^^
 
 warning: type `Priv1` is more private than the item `aliases_priv::f1`
-  --> $DIR/private-in-public.rs:133:5
+  --> $DIR/private-in-public.rs:134:5
    |
 LL |     pub fn f1(arg: PrivUseAlias) {}
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_priv::f1` is reachable at visibility `pub(crate)`
    |
 note: but type `Priv1` is only usable at visibility `pub(self)`
-  --> $DIR/private-in-public.rs:118:5
+  --> $DIR/private-in-public.rs:119:5
    |
 LL |     struct Priv1;
    |     ^^^^^^^^^^^^
 
 warning: type `Priv2` is more private than the item `aliases_priv::f2`
-  --> $DIR/private-in-public.rs:134:5
+  --> $DIR/private-in-public.rs:135:5
    |
 LL |     pub fn f2(arg: PrivAlias) {}
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_priv::f2` is reachable at visibility `pub(crate)`
    |
 note: but type `Priv2` is only usable at visibility `pub(self)`
-  --> $DIR/private-in-public.rs:119:5
+  --> $DIR/private-in-public.rs:120:5
    |
 LL |     struct Priv2;
    |     ^^^^^^^^^^^^
 
 warning: associated type `aliases_priv::PrivTr::Assoc` is more private than the item `aliases_priv::f3`
-  --> $DIR/private-in-public.rs:135:5
+  --> $DIR/private-in-public.rs:136:5
    |
 LL |     pub fn f3(arg: <Priv as PrivTr>::Assoc) {}
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_priv::f3` is reachable at visibility `pub(crate)`
    |
 note: but associated type `aliases_priv::PrivTr::Assoc` is only usable at visibility `pub(self)`
-  --> $DIR/private-in-public.rs:129:9
+  --> $DIR/private-in-public.rs:130:9
    |
 LL |         type Assoc = Priv3;
    |         ^^^^^^^^^^
 
+warning: trait `aliases_priv::PrivTr` is more private than the item `aliases_priv::f3`
+  --> $DIR/private-in-public.rs:136:5
+   |
+LL |     pub fn f3(arg: <Priv as PrivTr>::Assoc) {}
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_priv::f3` is reachable at visibility `pub(crate)`
+   |
+note: but trait `aliases_priv::PrivTr` is only usable at visibility `pub(self)`
+  --> $DIR/private-in-public.rs:129:5
+   |
+LL |     trait PrivTr {
+   |     ^^^^^^^^^^^^
+
 warning: type `aliases_priv::Priv` is more private than the item `aliases_priv::f3`
-  --> $DIR/private-in-public.rs:135:5
+  --> $DIR/private-in-public.rs:136:5
    |
 LL |     pub fn f3(arg: <Priv as PrivTr>::Assoc) {}
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_priv::f3` is reachable at visibility `pub(crate)`
    |
 note: but type `aliases_priv::Priv` is only usable at visibility `pub(self)`
-  --> $DIR/private-in-public.rs:116:5
+  --> $DIR/private-in-public.rs:117:5
    |
 LL |     struct Priv;
    |     ^^^^^^^^^^^
 
 warning: type `aliases_params::Priv` is more private than the item `aliases_params::f2`
-  --> $DIR/private-in-public.rs:145:5
+  --> $DIR/private-in-public.rs:147:5
    |
 LL |     pub fn f2(arg: PrivAliasGeneric) {}
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_params::f2` is reachable at visibility `pub(crate)`
    |
 note: but type `aliases_params::Priv` is only usable at visibility `pub(self)`
-  --> $DIR/private-in-public.rs:141:5
+  --> $DIR/private-in-public.rs:143:5
    |
 LL |     struct Priv;
    |     ^^^^^^^^^^^
 
 warning: type `aliases_params::Priv` is more private than the item `aliases_params::f3`
-  --> $DIR/private-in-public.rs:147:5
+  --> $DIR/private-in-public.rs:149:5
    |
 LL |     pub fn f3(arg: Result<u8>) {}
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^ function `aliases_params::f3` is reachable at visibility `pub(crate)`
    |
 note: but type `aliases_params::Priv` is only usable at visibility `pub(self)`
-  --> $DIR/private-in-public.rs:141:5
+  --> $DIR/private-in-public.rs:143:5
    |
 LL |     struct Priv;
    |     ^^^^^^^^^^^
 
-warning: 31 warnings emitted
+warning: 33 warnings emitted
 
-- 
cgit 1.4.1-3-g733a5


From 0f40b6545dc5acf1a3bfd5bcd97a9a9015e36a97 Mon Sep 17 00:00:00 2001
From: Urgau <urgau@numericable.fr>
Date: Fri, 8 Dec 2023 15:51:18 +0100
Subject: Remove extra check-cfg handled by libc directly

---
 src/bootstrap/src/lib.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs
index 60a89e9bf07..13391b1faa1 100644
--- a/src/bootstrap/src/lib.rs
+++ b/src/bootstrap/src/lib.rs
@@ -83,8 +83,6 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &str, Option<&[&'static str]>)] = &[
     (Some(Mode::Std), "no_global_oom_handling", None),
     (Some(Mode::Std), "no_rc", None),
     (Some(Mode::Std), "no_sync", None),
-    (Some(Mode::Std), "freebsd12", None),
-    (Some(Mode::Std), "freebsd13", None),
     (Some(Mode::Std), "backtrace_in_libstd", None),
     /* Extra values not defined in the built-in targets yet, but used in std */
     (Some(Mode::Std), "target_env", Some(&["libnx"])),
-- 
cgit 1.4.1-3-g733a5


From a0cbc168c9933663a714864b62f332386bcd1a0a Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Tue, 28 Nov 2023 03:18:56 +0000
Subject: Rework coroutine transform to be more flexible in preparation for
 async generators

---
 compiler/rustc_mir_transform/src/coroutine.rs | 203 ++++++++++++++++----------
 1 file changed, 123 insertions(+), 80 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index 79a1509531d..e0cfd7d0fcb 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -66,9 +66,9 @@ use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::dump_mir;
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
+use rustc_middle::ty::CoroutineArgs;
 use rustc_middle::ty::InstanceDef;
-use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt};
-use rustc_middle::ty::{CoroutineArgs, GenericArgsRef};
+use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_mir_dataflow::impls::{
     MaybeBorrowedLocals, MaybeLiveLocals, MaybeRequiresStorage, MaybeStorageLive,
 };
@@ -225,8 +225,6 @@ struct SuspensionPoint<'tcx> {
 struct TransformVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
     coroutine_kind: hir::CoroutineKind,
-    state_adt_ref: AdtDef<'tcx>,
-    state_args: GenericArgsRef<'tcx>,
 
     // The type of the discriminant in the coroutine struct
     discr_ty: Ty<'tcx>,
@@ -245,21 +243,34 @@ struct TransformVisitor<'tcx> {
     always_live_locals: BitSet<Local>,
 
     // The original RETURN_PLACE local
-    new_ret_local: Local,
+    old_ret_local: Local,
+
+    old_yield_ty: Ty<'tcx>,
+
+    old_ret_ty: Ty<'tcx>,
 }
 
 impl<'tcx> TransformVisitor<'tcx> {
     fn insert_none_ret_block(&self, body: &mut Body<'tcx>) -> BasicBlock {
-        let block = BasicBlock::new(body.basic_blocks.len());
+        assert!(matches!(self.coroutine_kind, CoroutineKind::Gen(_)));
 
+        let block = BasicBlock::new(body.basic_blocks.len());
         let source_info = SourceInfo::outermost(body.span);
+        let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
 
-        let (kind, idx) = self.coroutine_state_adt_and_variant_idx(true);
-        assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
         let statements = vec![Statement {
             kind: StatementKind::Assign(Box::new((
                 Place::return_place(),
-                Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
+                Rvalue::Aggregate(
+                    Box::new(AggregateKind::Adt(
+                        option_def_id,
+                        VariantIdx::from_usize(0),
+                        self.tcx.mk_args(&[self.old_yield_ty.into()]),
+                        None,
+                        None,
+                    )),
+                    IndexVec::new(),
+                ),
             ))),
             source_info,
         }];
@@ -273,23 +284,6 @@ impl<'tcx> TransformVisitor<'tcx> {
         block
     }
 
-    fn coroutine_state_adt_and_variant_idx(
-        &self,
-        is_return: bool,
-    ) -> (AggregateKind<'tcx>, VariantIdx) {
-        let idx = VariantIdx::new(match (is_return, self.coroutine_kind) {
-            (true, hir::CoroutineKind::Coroutine) => 1, // CoroutineState::Complete
-            (false, hir::CoroutineKind::Coroutine) => 0, // CoroutineState::Yielded
-            (true, hir::CoroutineKind::Async(_)) => 0,  // Poll::Ready
-            (false, hir::CoroutineKind::Async(_)) => 1, // Poll::Pending
-            (true, hir::CoroutineKind::Gen(_)) => 0,    // Option::None
-            (false, hir::CoroutineKind::Gen(_)) => 1,   // Option::Some
-        });
-
-        let kind = AggregateKind::Adt(self.state_adt_ref.did(), idx, self.state_args, None, None);
-        (kind, idx)
-    }
-
     // Make a `CoroutineState` or `Poll` variant assignment.
     //
     // `core::ops::CoroutineState` only has single element tuple variants,
@@ -302,51 +296,99 @@ impl<'tcx> TransformVisitor<'tcx> {
         is_return: bool,
         statements: &mut Vec<Statement<'tcx>>,
     ) {
-        let (kind, idx) = self.coroutine_state_adt_and_variant_idx(is_return);
-
-        match self.coroutine_kind {
-            // `Poll::Pending`
+        let rvalue = match self.coroutine_kind {
             CoroutineKind::Async(_) => {
-                if !is_return {
-                    assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
-
-                    // FIXME(swatinem): assert that `val` is indeed unit?
-                    statements.push(Statement {
-                        kind: StatementKind::Assign(Box::new((
-                            Place::return_place(),
-                            Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
-                        ))),
-                        source_info,
-                    });
-                    return;
+                let poll_def_id = self.tcx.require_lang_item(LangItem::Poll, None);
+                let args = self.tcx.mk_args(&[self.old_ret_ty.into()]);
+                if is_return {
+                    // Poll::Ready(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            poll_def_id,
+                            VariantIdx::from_usize(0),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
+                } else {
+                    // Poll::Pending
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            poll_def_id,
+                            VariantIdx::from_usize(1),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::new(),
+                    )
                 }
             }
-            // `Option::None`
             CoroutineKind::Gen(_) => {
+                let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
+                let args = self.tcx.mk_args(&[self.old_yield_ty.into()]);
                 if is_return {
-                    assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
-
-                    statements.push(Statement {
-                        kind: StatementKind::Assign(Box::new((
-                            Place::return_place(),
-                            Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
-                        ))),
-                        source_info,
-                    });
-                    return;
+                    // None
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            option_def_id,
+                            VariantIdx::from_usize(0),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::new(),
+                    )
+                } else {
+                    // Some(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            option_def_id,
+                            VariantIdx::from_usize(1),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
                 }
             }
-            CoroutineKind::Coroutine => {}
-        }
-
-        // else: `Poll::Ready(x)`, `CoroutineState::Yielded(x)`, `CoroutineState::Complete(x)`, or `Option::Some(x)`
-        assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 1);
+            CoroutineKind::Coroutine => {
+                let coroutine_state_def_id =
+                    self.tcx.require_lang_item(LangItem::CoroutineState, None);
+                let args = self.tcx.mk_args(&[self.old_yield_ty.into(), self.old_ret_ty.into()]);
+                if is_return {
+                    // CoroutineState::Complete(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            coroutine_state_def_id,
+                            VariantIdx::from_usize(1),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
+                } else {
+                    // CoroutineState::Yielded(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            coroutine_state_def_id,
+                            VariantIdx::from_usize(0),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
+                }
+            }
+        };
 
         statements.push(Statement {
-            kind: StatementKind::Assign(Box::new((
-                Place::return_place(),
-                Rvalue::Aggregate(Box::new(kind), [val].into()),
-            ))),
+            kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
             source_info,
         });
     }
@@ -420,7 +462,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> {
 
         let ret_val = match data.terminator().kind {
             TerminatorKind::Return => {
-                Some((true, None, Operand::Move(Place::from(self.new_ret_local)), None))
+                Some((true, None, Operand::Move(Place::from(self.old_ret_local)), None))
             }
             TerminatorKind::Yield { ref value, resume, resume_arg, drop } => {
                 Some((false, Some((resume, resume_arg)), value.clone(), drop))
@@ -1493,10 +1535,11 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>(
 
 impl<'tcx> MirPass<'tcx> for StateTransform {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let Some(yield_ty) = body.yield_ty() else {
+        let Some(old_yield_ty) = body.yield_ty() else {
             // This only applies to coroutines
             return;
         };
+        let old_ret_ty = body.return_ty();
 
         assert!(body.coroutine_drop().is_none());
 
@@ -1520,34 +1563,33 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
 
         let is_async_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::Async(_)));
         let is_gen_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::Gen(_)));
-        let (state_adt_ref, state_args) = match body.coroutine_kind().unwrap() {
+        let new_ret_ty = match body.coroutine_kind().unwrap() {
             CoroutineKind::Async(_) => {
                 // Compute Poll<return_ty>
                 let poll_did = tcx.require_lang_item(LangItem::Poll, None);
                 let poll_adt_ref = tcx.adt_def(poll_did);
-                let poll_args = tcx.mk_args(&[body.return_ty().into()]);
-                (poll_adt_ref, poll_args)
+                let poll_args = tcx.mk_args(&[old_ret_ty.into()]);
+                Ty::new_adt(tcx, poll_adt_ref, poll_args)
             }
             CoroutineKind::Gen(_) => {
                 // Compute Option<yield_ty>
                 let option_did = tcx.require_lang_item(LangItem::Option, None);
                 let option_adt_ref = tcx.adt_def(option_did);
-                let option_args = tcx.mk_args(&[body.yield_ty().unwrap().into()]);
-                (option_adt_ref, option_args)
+                let option_args = tcx.mk_args(&[old_yield_ty.into()]);
+                Ty::new_adt(tcx, option_adt_ref, option_args)
             }
             CoroutineKind::Coroutine => {
                 // Compute CoroutineState<yield_ty, return_ty>
                 let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
                 let state_adt_ref = tcx.adt_def(state_did);
-                let state_args = tcx.mk_args(&[yield_ty.into(), body.return_ty().into()]);
-                (state_adt_ref, state_args)
+                let state_args = tcx.mk_args(&[old_yield_ty.into(), old_ret_ty.into()]);
+                Ty::new_adt(tcx, state_adt_ref, state_args)
             }
         };
-        let ret_ty = Ty::new_adt(tcx, state_adt_ref, state_args);
 
-        // We rename RETURN_PLACE which has type mir.return_ty to new_ret_local
+        // We rename RETURN_PLACE which has type mir.return_ty to old_ret_local
         // RETURN_PLACE then is a fresh unused local with type ret_ty.
-        let new_ret_local = replace_local(RETURN_PLACE, ret_ty, body, tcx);
+        let old_ret_local = replace_local(RETURN_PLACE, new_ret_ty, body, tcx);
 
         // Replace all occurrences of `ResumeTy` with `&mut Context<'_>` within async bodies.
         if is_async_kind {
@@ -1564,9 +1606,10 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         } else {
             body.local_decls[resume_local].ty
         };
-        let new_resume_local = replace_local(resume_local, resume_ty, body, tcx);
+        let old_resume_local = replace_local(resume_local, resume_ty, body, tcx);
 
-        // When first entering the coroutine, move the resume argument into its new local.
+        // When first entering the coroutine, move the resume argument into its old local
+        // (which is now a generator interior).
         let source_info = SourceInfo::outermost(body.span);
         let stmts = &mut body.basic_blocks_mut()[START_BLOCK].statements;
         stmts.insert(
@@ -1574,7 +1617,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
             Statement {
                 source_info,
                 kind: StatementKind::Assign(Box::new((
-                    new_resume_local.into(),
+                    old_resume_local.into(),
                     Rvalue::Use(Operand::Move(resume_local.into())),
                 ))),
             },
@@ -1610,14 +1653,14 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         let mut transform = TransformVisitor {
             tcx,
             coroutine_kind: body.coroutine_kind().unwrap(),
-            state_adt_ref,
-            state_args,
             remap,
             storage_liveness,
             always_live_locals,
             suspension_points: Vec::new(),
-            new_ret_local,
+            old_ret_local,
             discr_ty,
+            old_ret_ty,
+            old_yield_ty,
         };
         transform.visit_body(body);
 
-- 
cgit 1.4.1-3-g733a5


From 96bb542a3171a7cbf23cac45d4b9adc7b05b5317 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Tue, 28 Nov 2023 18:18:19 +0000
Subject: Implement `async gen` blocks

---
 compiler/rustc_ast/src/ast.rs                      |   2 +
 compiler/rustc_ast_lowering/src/expr.rs            | 141 +++++++++++++++++++--
 .../src/diagnostics/conflict_errors.rs             |  15 ++-
 .../rustc_borrowck/src/diagnostics/region_name.rs  |  19 ++-
 .../rustc_codegen_ssa/src/debuginfo/type_names.rs  |   3 +
 compiler/rustc_codegen_ssa/src/mir/locals.rs       |   6 +-
 compiler/rustc_hir/src/hir.rs                      |  25 ++--
 compiler/rustc_hir/src/lang_items.rs               |   5 +
 compiler/rustc_hir_typeck/src/check.rs             |   4 +-
 compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs     |   1 +
 compiler/rustc_middle/src/mir/terminator.rs        |   8 ++
 compiler/rustc_middle/src/traits/select.rs         |   8 +-
 compiler/rustc_middle/src/ty/context.rs            |   7 +-
 compiler/rustc_middle/src/ty/util.rs               |   2 +
 compiler/rustc_mir_transform/src/coroutine.rs      |  30 ++++-
 compiler/rustc_parse/src/parser/expr.rs            |  23 ++--
 compiler/rustc_parse/src/parser/item.rs            |   6 +-
 compiler/rustc_smir/src/rustc_smir/convert/mod.rs  |   1 +
 compiler/rustc_span/src/symbol.rs                  |   5 +
 .../src/solve/assembly/mod.rs                      |   7 +
 .../src/solve/normalizes_to/mod.rs                 |  34 +++++
 .../rustc_trait_selection/src/solve/trait_goals.rs |  24 ++++
 .../src/traits/error_reporting/suggestions.rs      |  21 ++-
 .../traits/error_reporting/type_err_ctxt_ext.rs    |   3 +
 .../rustc_trait_selection/src/traits/project.rs    |  66 +++++++++-
 .../src/traits/select/candidate_assembly.rs        |  30 +++++
 .../src/traits/select/confirmation.rs              |  34 +++++
 .../rustc_trait_selection/src/traits/select/mod.rs |   6 +
 compiler/rustc_trait_selection/src/traits/util.rs  |  11 ++
 compiler/rustc_ty_utils/src/abi.rs                 |  30 ++++-
 compiler/rustc_ty_utils/src/instance.rs            |  15 +++
 library/core/src/async_iter/async_iter.rs          |  25 ++++
 32 files changed, 563 insertions(+), 54 deletions(-)

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index d6c2bfacf66..7cd1ff03400 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -1516,6 +1516,7 @@ pub enum ExprKind {
 pub enum GenBlockKind {
     Async,
     Gen,
+    AsyncGen,
 }
 
 impl fmt::Display for GenBlockKind {
@@ -1529,6 +1530,7 @@ impl GenBlockKind {
         match self {
             GenBlockKind::Async => "async",
             GenBlockKind::Gen => "gen",
+            GenBlockKind::AsyncGen => "async gen",
         }
     }
 }
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 3556ee02bd7..a8cbb42997d 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -324,6 +324,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     hir::CoroutineSource::Block,
                     |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
                 ),
+                ExprKind::Gen(capture_clause, block, GenBlockKind::AsyncGen) => self
+                    .make_async_gen_expr(
+                        *capture_clause,
+                        e.id,
+                        None,
+                        e.span,
+                        hir::CoroutineSource::Block,
+                        |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
+                    ),
                 ExprKind::Yield(opt_expr) => self.lower_expr_yield(e.span, opt_expr.as_deref()),
                 ExprKind::Err => hir::ExprKind::Err(
                     self.tcx.sess.span_delayed_bug(e.span, "lowered ExprKind::Err"),
@@ -706,6 +715,87 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }))
     }
 
+    /// Lower a `async gen` construct to a generator that implements `AsyncIterator`.
+    ///
+    /// This results in:
+    ///
+    /// ```text
+    /// static move? |_task_context| -> () {
+    ///     <body>
+    /// }
+    /// ```
+    pub(super) fn make_async_gen_expr(
+        &mut self,
+        capture_clause: CaptureBy,
+        closure_node_id: NodeId,
+        _yield_ty: Option<hir::FnRetTy<'hir>>,
+        span: Span,
+        async_coroutine_source: hir::CoroutineSource,
+        body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
+    ) -> hir::ExprKind<'hir> {
+        let output = hir::FnRetTy::DefaultReturn(self.lower_span(span));
+
+        // Resume argument type: `ResumeTy`
+        let unstable_span = self.mark_span_with_reason(
+            DesugaringKind::Async,
+            span,
+            Some(self.allow_gen_future.clone()),
+        );
+        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
+        let input_ty = hir::Ty {
+            hir_id: self.next_id(),
+            kind: hir::TyKind::Path(resume_ty),
+            span: unstable_span,
+        };
+
+        // The closure/coroutine `FnDecl` takes a single (resume) argument of type `input_ty`.
+        let fn_decl = self.arena.alloc(hir::FnDecl {
+            inputs: arena_vec![self; input_ty],
+            output,
+            c_variadic: false,
+            implicit_self: hir::ImplicitSelfKind::None,
+            lifetime_elision_allowed: false,
+        });
+
+        // Lower the argument pattern/ident. The ident is used again in the `.await` lowering.
+        let (pat, task_context_hid) = self.pat_ident_binding_mode(
+            span,
+            Ident::with_dummy_span(sym::_task_context),
+            hir::BindingAnnotation::MUT,
+        );
+        let param = hir::Param {
+            hir_id: self.next_id(),
+            pat,
+            ty_span: self.lower_span(span),
+            span: self.lower_span(span),
+        };
+        let params = arena_vec![self; param];
+
+        let body = self.lower_body(move |this| {
+            this.coroutine_kind = Some(hir::CoroutineKind::AsyncGen(async_coroutine_source));
+
+            let old_ctx = this.task_context;
+            this.task_context = Some(task_context_hid);
+            let res = body(this);
+            this.task_context = old_ctx;
+            (params, res)
+        });
+
+        // `static |_task_context| -> <ret_ty> { body }`:
+        hir::ExprKind::Closure(self.arena.alloc(hir::Closure {
+            def_id: self.local_def_id(closure_node_id),
+            binder: hir::ClosureBinder::Default,
+            capture_clause,
+            bound_generic_params: &[],
+            fn_decl,
+            body,
+            fn_decl_span: self.lower_span(span),
+            fn_arg_span: None,
+            movability: Some(hir::Movability::Static),
+            constness: hir::Constness::NotConst,
+        }))
+    }
+
     /// Forwards a possible `#[track_caller]` annotation from `outer_hir_id` to
     /// `inner_hir_id` in case the `async_fn_track_caller` feature is enabled.
     pub(super) fn maybe_forward_track_caller(
@@ -755,15 +845,18 @@ impl<'hir> LoweringContext<'_, 'hir> {
     /// ```
     fn lower_expr_await(&mut self, await_kw_span: Span, expr: &Expr) -> hir::ExprKind<'hir> {
         let full_span = expr.span.to(await_kw_span);
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Async(_)) => {}
+
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Async(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Coroutine) | Some(hir::CoroutineKind::Gen(_)) | None => {
                 return hir::ExprKind::Err(self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
                     await_kw_span,
                     item_span: self.current_item,
                 }));
             }
-        }
+        };
+
         let span = self.mark_span_with_reason(DesugaringKind::Await, await_kw_span, None);
         let gen_future_span = self.mark_span_with_reason(
             DesugaringKind::Await,
@@ -852,12 +945,19 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.stmt_expr(span, match_expr)
         };
 
-        // task_context = yield ();
+        // Depending on `async` of `async gen`:
+        // async     - task_context = yield ();
+        // async gen - task_context = yield ASYNC_GEN_PENDING;
         let yield_stmt = {
-            let unit = self.expr_unit(span);
+            let yielded = if is_async_gen {
+                self.arena.alloc(self.expr_lang_item_path(span, hir::LangItem::AsyncGenPending))
+            } else {
+                self.expr_unit(span)
+            };
+
             let yield_expr = self.expr(
                 span,
-                hir::ExprKind::Yield(unit, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
+                hir::ExprKind::Yield(yielded, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
             );
             let yield_expr = self.arena.alloc(yield_expr);
 
@@ -967,7 +1067,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
                 Some(movability)
             }
-            Some(hir::CoroutineKind::Gen(_)) | Some(hir::CoroutineKind::Async(_)) => {
+            Some(
+                hir::CoroutineKind::Gen(_)
+                | hir::CoroutineKind::Async(_)
+                | hir::CoroutineKind::AsyncGen(_),
+            ) => {
                 panic!("non-`async`/`gen` closure body turned `async`/`gen` during lowering");
             }
             None => {
@@ -1474,8 +1578,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_expr_yield(&mut self, span: Span, opt_expr: Option<&Expr>) -> hir::ExprKind<'hir> {
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Gen(_)) => {}
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Gen(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Async(_)) => {
                 return hir::ExprKind::Err(
                     self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span }),
@@ -1491,14 +1596,24 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     )
                     .emit();
                 }
-                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine)
+                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine);
+                false
             }
-        }
+        };
 
-        let expr =
+        let mut yielded =
             opt_expr.as_ref().map(|x| self.lower_expr(x)).unwrap_or_else(|| self.expr_unit(span));
 
-        hir::ExprKind::Yield(expr, hir::YieldSource::Yield)
+        if is_async_gen {
+            // yield async_gen_ready($expr);
+            yielded = self.expr_call_lang_item_fn(
+                span,
+                hir::LangItem::AsyncGenReady,
+                std::slice::from_ref(yielded),
+            );
+        }
+
+        hir::ExprKind::Yield(yielded, hir::YieldSource::Yield)
     }
 
     /// Desugar `ExprForLoop` from: `[opt_ident]: for <pat> in <head> <body>` into:
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 7bcad92ff33..7e62bb9793d 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -2517,12 +2517,23 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 CoroutineKind::Gen(kind) => match kind {
                     CoroutineSource::Block => "gen block",
                     CoroutineSource::Closure => "gen closure",
-                    _ => bug!("gen block/closure expected, but gen function found."),
+                    CoroutineSource::Fn => {
+                        bug!("gen block/closure expected, but gen function found.")
+                    }
+                },
+                CoroutineKind::AsyncGen(kind) => match kind {
+                    CoroutineSource::Block => "async gen block",
+                    CoroutineSource::Closure => "async gen closure",
+                    CoroutineSource::Fn => {
+                        bug!("gen block/closure expected, but gen function found.")
+                    }
                 },
                 CoroutineKind::Async(async_kind) => match async_kind {
                     CoroutineSource::Block => "async block",
                     CoroutineSource::Closure => "async closure",
-                    _ => bug!("async block/closure expected, but async function found."),
+                    CoroutineSource::Fn => {
+                        bug!("async block/closure expected, but async function found.")
+                    }
                 },
                 CoroutineKind::Coroutine => "coroutine",
             },
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index 977a5d5d50d..a17c3bc3a78 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -684,7 +684,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                     hir::FnRetTy::Return(hir_ty) => (fn_decl.output.span(), Some(hir_ty)),
                 };
                 let mir_description = match hir.body(body).coroutine_kind {
-                    Some(hir::CoroutineKind::Async(gen)) => match gen {
+                    Some(hir::CoroutineKind::Async(src)) => match src {
                         hir::CoroutineSource::Block => " of async block",
                         hir::CoroutineSource::Closure => " of async closure",
                         hir::CoroutineSource::Fn => {
@@ -701,7 +701,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                             " of async function"
                         }
                     },
-                    Some(hir::CoroutineKind::Gen(gen)) => match gen {
+                    Some(hir::CoroutineKind::Gen(src)) => match src {
                         hir::CoroutineSource::Block => " of gen block",
                         hir::CoroutineSource::Closure => " of gen closure",
                         hir::CoroutineSource::Fn => {
@@ -715,6 +715,21 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                             " of gen function"
                         }
                     },
+
+                    Some(hir::CoroutineKind::AsyncGen(src)) => match src {
+                        hir::CoroutineSource::Block => " of async gen block",
+                        hir::CoroutineSource::Closure => " of async gen closure",
+                        hir::CoroutineSource::Fn => {
+                            let parent_item =
+                                hir.get_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                            let output = &parent_item
+                                .fn_decl()
+                                .expect("coroutine lowered from async gen fn should be in fn")
+                                .output;
+                            span = output.span();
+                            " of async gen function"
+                        }
+                    },
                     Some(hir::CoroutineKind::Coroutine) => " of coroutine",
                     None => " of closure",
                 };
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 8630e5623e1..dda30046bfb 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -566,6 +566,9 @@ fn coroutine_kind_label(coroutine_kind: Option<CoroutineKind>) -> &'static str {
         Some(CoroutineKind::Async(CoroutineSource::Block)) => "async_block",
         Some(CoroutineKind::Async(CoroutineSource::Closure)) => "async_closure",
         Some(CoroutineKind::Async(CoroutineSource::Fn)) => "async_fn",
+        Some(CoroutineKind::AsyncGen(CoroutineSource::Block)) => "async_gen_block",
+        Some(CoroutineKind::AsyncGen(CoroutineSource::Closure)) => "async_gen_closure",
+        Some(CoroutineKind::AsyncGen(CoroutineSource::Fn)) => "async_gen_fn",
         Some(CoroutineKind::Coroutine) => "coroutine",
         None => "closure",
     }
diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs
index 378c5401322..7db260c9f5b 100644
--- a/compiler/rustc_codegen_ssa/src/mir/locals.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs
@@ -43,7 +43,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     let local = mir::Local::from_usize(local);
                     let expected_ty = self.monomorphize(self.mir.local_decls[local].ty);
                     if expected_ty != op.layout.ty {
-                        warn!("Unexpected initial operand type. See the issues/114858");
+                        warn!(
+                            "Unexpected initial operand type: expected {expected_ty:?}, found {:?}.\
+                            See <https://github.com/rust-lang/rust/issues/114858>.",
+                            op.layout.ty
+                        );
                     }
                 }
             }
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index ee66ebc2554..e2aef43dcf0 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -1339,12 +1339,16 @@ impl<'hir> Body<'hir> {
 /// The type of source expression that caused this coroutine to be created.
 #[derive(Clone, PartialEq, Eq, Debug, Copy, Hash, HashStable_Generic, Encodable, Decodable)]
 pub enum CoroutineKind {
-    /// An explicit `async` block or the body of an async function.
+    /// An explicit `async` block or the body of an `async` function.
     Async(CoroutineSource),
 
     /// An explicit `gen` block or the body of a `gen` function.
     Gen(CoroutineSource),
 
+    /// An explicit `async gen` block or the body of an `async gen` function,
+    /// which is able to both `yield` and `.await`.
+    AsyncGen(CoroutineSource),
+
     /// A coroutine literal created via a `yield` inside a closure.
     Coroutine,
 }
@@ -1369,6 +1373,14 @@ impl fmt::Display for CoroutineKind {
                 }
                 k.fmt(f)
             }
+            CoroutineKind::AsyncGen(k) => {
+                if f.alternate() {
+                    f.write_str("`async gen` ")?;
+                } else {
+                    f.write_str("async gen ")?
+                }
+                k.fmt(f)
+            }
         }
     }
 }
@@ -2064,17 +2076,6 @@ impl fmt::Display for YieldSource {
     }
 }
 
-impl From<CoroutineKind> for YieldSource {
-    fn from(kind: CoroutineKind) -> Self {
-        match kind {
-            // Guess based on the kind of the current coroutine.
-            CoroutineKind::Coroutine => Self::Yield,
-            CoroutineKind::Async(_) => Self::Await { expr: None },
-            CoroutineKind::Gen(_) => Self::Yield,
-        }
-    }
-}
-
 // N.B., if you change this, you'll probably want to change the corresponding
 // type structure in middle/ty.rs as well.
 #[derive(Debug, Clone, Copy, HashStable_Generic)]
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 60f1449c177..b0b53bb7478 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -212,6 +212,7 @@ language_item_table! {
 
     Iterator,                sym::iterator,            iterator_trait,             Target::Trait,          GenericRequirement::Exact(0);
     Future,                  sym::future_trait,        future_trait,               Target::Trait,          GenericRequirement::Exact(0);
+    AsyncIterator,           sym::async_iterator,      async_iterator_trait,       Target::Trait,          GenericRequirement::Exact(0);
     CoroutineState,          sym::coroutine_state,     coroutine_state,            Target::Enum,           GenericRequirement::None;
     Coroutine,               sym::coroutine,           coroutine_trait,            Target::Trait,          GenericRequirement::Minimum(1);
     Unpin,                   sym::unpin,               unpin_trait,                Target::Trait,          GenericRequirement::None;
@@ -294,6 +295,10 @@ language_item_table! {
     PollReady,               sym::Ready,               poll_ready_variant,         Target::Variant,        GenericRequirement::None;
     PollPending,             sym::Pending,             poll_pending_variant,       Target::Variant,        GenericRequirement::None;
 
+    AsyncGenReady,           sym::AsyncGenReady,       async_gen_ready,            Target::Method(MethodKind::Inherent), GenericRequirement::Exact(1);
+    AsyncGenPending,         sym::AsyncGenPending,     async_gen_pending,          Target::AssocConst,     GenericRequirement::Exact(1);
+    AsyncGenFinished,        sym::AsyncGenFinished,    async_gen_finished,         Target::AssocConst,     GenericRequirement::Exact(1);
+
     // FIXME(swatinem): the following lang items are used for async lowering and
     // should become obsolete eventually.
     ResumeTy,                sym::ResumeTy,            resume_ty,                  Target::Struct,         GenericRequirement::None;
diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs
index 0cd1ae2dbfe..91c96d6f76f 100644
--- a/compiler/rustc_hir_typeck/src/check.rs
+++ b/compiler/rustc_hir_typeck/src/check.rs
@@ -59,7 +59,9 @@ pub(super) fn check_fn<'a, 'tcx>(
         && can_be_coroutine.is_some()
     {
         let yield_ty = match kind {
-            hir::CoroutineKind::Gen(..) | hir::CoroutineKind::Coroutine => {
+            hir::CoroutineKind::Gen(..)
+            | hir::CoroutineKind::AsyncGen(..)
+            | hir::CoroutineKind::Coroutine => {
                 let yield_ty = fcx.next_ty_var(TypeVariableOrigin {
                     kind: TypeVariableOriginKind::TypeInference,
                     span,
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
index 1f2bd92a15c..df840aaa578 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
@@ -763,6 +763,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let args = self.fresh_args_for_item(span, def_id);
         let ty = item_ty.instantiate(self.tcx, args);
 
+        self.write_args(hir_id, args);
         self.write_resolution(hir_id, Ok((def_kind, def_id)));
 
         let code = match lang_item {
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 9a6ac6ff57a..aa4cb36c5ce 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -150,11 +150,17 @@ impl<O> AssertKind<O> {
             RemainderByZero(_) => "attempt to calculate the remainder with a divisor of zero",
             ResumedAfterReturn(CoroutineKind::Coroutine) => "coroutine resumed after completion",
             ResumedAfterReturn(CoroutineKind::Async(_)) => "`async fn` resumed after completion",
+            ResumedAfterReturn(CoroutineKind::AsyncGen(_)) => {
+                "`async gen fn` resumed after completion"
+            }
             ResumedAfterReturn(CoroutineKind::Gen(_)) => {
                 "`gen fn` should just keep returning `None` after completion"
             }
             ResumedAfterPanic(CoroutineKind::Coroutine) => "coroutine resumed after panicking",
             ResumedAfterPanic(CoroutineKind::Async(_)) => "`async fn` resumed after panicking",
+            ResumedAfterPanic(CoroutineKind::AsyncGen(_)) => {
+                "`async gen fn` resumed after panicking"
+            }
             ResumedAfterPanic(CoroutineKind::Gen(_)) => {
                 "`gen fn` should just keep returning `None` after panicking"
             }
@@ -245,6 +251,7 @@ impl<O> AssertKind<O> {
             DivisionByZero(_) => middle_assert_divide_by_zero,
             RemainderByZero(_) => middle_assert_remainder_by_zero,
             ResumedAfterReturn(CoroutineKind::Async(_)) => middle_assert_async_resume_after_return,
+            ResumedAfterReturn(CoroutineKind::AsyncGen(_)) => todo!(),
             ResumedAfterReturn(CoroutineKind::Gen(_)) => {
                 bug!("gen blocks can be resumed after they return and will keep returning `None`")
             }
@@ -252,6 +259,7 @@ impl<O> AssertKind<O> {
                 middle_assert_coroutine_resume_after_return
             }
             ResumedAfterPanic(CoroutineKind::Async(_)) => middle_assert_async_resume_after_panic,
+            ResumedAfterPanic(CoroutineKind::AsyncGen(_)) => todo!(),
             ResumedAfterPanic(CoroutineKind::Gen(_)) => middle_assert_gen_resume_after_panic,
             ResumedAfterPanic(CoroutineKind::Coroutine) => {
                 middle_assert_coroutine_resume_after_panic
diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs
index 96ed1a4d0be..e8e2907eb33 100644
--- a/compiler/rustc_middle/src/traits/select.rs
+++ b/compiler/rustc_middle/src/traits/select.rs
@@ -144,10 +144,14 @@ pub enum SelectionCandidate<'tcx> {
     /// generated for an async construct.
     FutureCandidate,
 
-    /// Implementation of an `Iterator` trait by one of the generator types
-    /// generated for a gen construct.
+    /// Implementation of an `Iterator` trait by one of the coroutine types
+    /// generated for a `gen` construct.
     IteratorCandidate,
 
+    /// Implementation of an `AsyncIterator` trait by one of the coroutine types
+    /// generated for a `async gen` construct.
+    AsyncIteratorCandidate,
+
     /// Implementation of a `Fn`-family trait by one of the anonymous
     /// types generated for a fn pointer type (e.g., `fn(int) -> int`)
     FnPointerCandidate {
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 3012434ad3f..6ebfe778e7f 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -825,11 +825,16 @@ impl<'tcx> TyCtxt<'tcx> {
         matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Coroutine))
     }
 
-    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a gen construct.
+    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a `gen` construct.
     pub fn coroutine_is_gen(self, def_id: DefId) -> bool {
         matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Gen(_)))
     }
 
+    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a `async gen` construct.
+    pub fn coroutine_is_async_gen(self, def_id: DefId) -> bool {
+        matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::AsyncGen(_)))
+    }
+
     pub fn stability(self) -> &'tcx stability::Index {
         self.stability_index(())
     }
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 52c3529d2b4..b7c3edee9e5 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -732,6 +732,7 @@ impl<'tcx> TyCtxt<'tcx> {
             DefKind::Closure if let Some(coroutine_kind) = self.coroutine_kind(def_id) => {
                 match coroutine_kind {
                     rustc_hir::CoroutineKind::Async(..) => "async closure",
+                    rustc_hir::CoroutineKind::AsyncGen(..) => "async gen closure",
                     rustc_hir::CoroutineKind::Coroutine => "coroutine",
                     rustc_hir::CoroutineKind::Gen(..) => "gen closure",
                 }
@@ -752,6 +753,7 @@ impl<'tcx> TyCtxt<'tcx> {
             DefKind::Closure if let Some(coroutine_kind) = self.coroutine_kind(def_id) => {
                 match coroutine_kind {
                     rustc_hir::CoroutineKind::Async(..) => "an",
+                    rustc_hir::CoroutineKind::AsyncGen(..) => "an",
                     rustc_hir::CoroutineKind::Coroutine => "a",
                     rustc_hir::CoroutineKind::Gen(..) => "a",
                 }
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index e0cfd7d0fcb..2b591abb05d 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -355,6 +355,26 @@ impl<'tcx> TransformVisitor<'tcx> {
                     )
                 }
             }
+            CoroutineKind::AsyncGen(_) => {
+                if is_return {
+                    let ty::Adt(_poll_adt, args) = *self.old_yield_ty.kind() else { bug!() };
+                    let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else { bug!() };
+                    let yield_ty = args.type_at(0);
+                    Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                        span: source_info.span,
+                        const_: Const::Unevaluated(
+                            UnevaluatedConst::new(
+                                self.tcx.require_lang_item(LangItem::AsyncGenFinished, None),
+                                self.tcx.mk_args(&[yield_ty.into()]),
+                            ),
+                            self.old_yield_ty,
+                        ),
+                        user_ty: None,
+                    })))
+                } else {
+                    Rvalue::Use(val)
+                }
+            }
             CoroutineKind::Coroutine => {
                 let coroutine_state_def_id =
                     self.tcx.require_lang_item(LangItem::CoroutineState, None);
@@ -1373,7 +1393,8 @@ fn create_coroutine_resume_function<'tcx>(
 
     if can_return {
         let block = match coroutine_kind {
-            CoroutineKind::Async(_) | CoroutineKind::Coroutine => {
+            // FIXME(gen_blocks): Should `async gen` yield `None` when resumed once again?
+            CoroutineKind::Async(_) | CoroutineKind::AsyncGen(_) | CoroutineKind::Coroutine => {
                 insert_panic_block(tcx, body, ResumedAfterReturn(coroutine_kind))
             }
             CoroutineKind::Gen(_) => transform.insert_none_ret_block(body),
@@ -1562,6 +1583,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         };
 
         let is_async_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::Async(_)));
+        let is_async_gen_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::AsyncGen(_)));
         let is_gen_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::Gen(_)));
         let new_ret_ty = match body.coroutine_kind().unwrap() {
             CoroutineKind::Async(_) => {
@@ -1578,6 +1600,10 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
                 let option_args = tcx.mk_args(&[old_yield_ty.into()]);
                 Ty::new_adt(tcx, option_adt_ref, option_args)
             }
+            CoroutineKind::AsyncGen(_) => {
+                // The yield ty is already `Poll<Option<yield_ty>>`
+                old_yield_ty
+            }
             CoroutineKind::Coroutine => {
                 // Compute CoroutineState<yield_ty, return_ty>
                 let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
@@ -1592,7 +1618,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         let old_ret_local = replace_local(RETURN_PLACE, new_ret_ty, body, tcx);
 
         // Replace all occurrences of `ResumeTy` with `&mut Context<'_>` within async bodies.
-        if is_async_kind {
+        if is_async_kind || is_async_gen_kind {
             transform_async_context(tcx, body);
         }
 
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 8482824ec4b..b5a9e91ae94 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1442,8 +1442,9 @@ impl<'a> Parser<'a> {
             } else if this.token.uninterpolated_span().at_least_rust_2018() {
                 // `Span:.at_least_rust_2018()` is somewhat expensive; don't get it repeatedly.
                 if this.check_keyword(kw::Async) {
-                    if this.is_gen_block(kw::Async) {
-                        // Check for `async {` and `async move {`.
+                    if this.is_gen_block(kw::Async, 0) || this.is_gen_block(kw::Gen, 1) {
+                        // Check for `async {` and `async move {`,
+                        // or `async gen {` and `async gen move {`.
                         this.parse_gen_block()
                     } else {
                         this.parse_expr_closure()
@@ -1451,7 +1452,7 @@ impl<'a> Parser<'a> {
                 } else if this.eat_keyword(kw::Await) {
                     this.recover_incorrect_await_syntax(lo, this.prev_token.span)
                 } else if this.token.uninterpolated_span().at_least_rust_2024() {
-                    if this.is_gen_block(kw::Gen) {
+                    if this.is_gen_block(kw::Gen, 0) {
                         this.parse_gen_block()
                     } else {
                         this.parse_expr_lit()
@@ -3179,7 +3180,7 @@ impl<'a> Parser<'a> {
     fn parse_gen_block(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
         let kind = if self.eat_keyword(kw::Async) {
-            GenBlockKind::Async
+            if self.eat_keyword(kw::Gen) { GenBlockKind::AsyncGen } else { GenBlockKind::Async }
         } else {
             assert!(self.eat_keyword(kw::Gen));
             self.sess.gated_spans.gate(sym::gen_blocks, lo.to(self.token.span));
@@ -3191,22 +3192,26 @@ impl<'a> Parser<'a> {
         Ok(self.mk_expr_with_attrs(lo.to(self.prev_token.span), kind, attrs))
     }
 
-    fn is_gen_block(&self, kw: Symbol) -> bool {
-        self.token.is_keyword(kw)
+    fn is_gen_block(&self, kw: Symbol, lookahead: usize) -> bool {
+        self.is_keyword_ahead(lookahead, &[kw])
             && ((
                 // `async move {`
-                self.is_keyword_ahead(1, &[kw::Move])
-                    && self.look_ahead(2, |t| {
+                self.is_keyword_ahead(lookahead + 1, &[kw::Move])
+                    && self.look_ahead(lookahead + 2, |t| {
                         *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()
                     })
             ) || (
                 // `async {`
-                self.look_ahead(1, |t| {
+                self.look_ahead(lookahead + 1, |t| {
                     *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()
                 })
             ))
     }
 
+    pub(super) fn is_async_gen_block(&self) -> bool {
+        self.token.is_keyword(kw::Async) && self.is_gen_block(kw::Gen, 1)
+    }
+
     fn is_certainly_not_a_block(&self) -> bool {
         self.look_ahead(1, |t| t.is_ident())
             && (
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 086e8d5cf9b..5fa9efb5af7 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -2359,8 +2359,10 @@ impl<'a> Parser<'a> {
                             || case == Case::Insensitive
                                 && t.is_non_raw_ident_where(|i| quals.iter().any(|qual| qual.as_str() == i.name.as_str().to_lowercase()))
                         )
-                        // Rule out unsafe extern block.
-                        && !self.is_unsafe_foreign_mod())
+                        // Rule out `unsafe extern {`.
+                        && !self.is_unsafe_foreign_mod()
+                        // Rule out `async gen {` and `async gen move {`
+                        && !self.is_async_gen_block())
                 })
             // `extern ABI fn`
             || self.check_keyword_case(kw::Extern, case)
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mod.rs b/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
index 9c0b2b29bca..ce575f269ca 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
@@ -56,6 +56,7 @@ impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineKind {
                 stable_mir::mir::CoroutineKind::Gen(source.stable(tables))
             }
             CoroutineKind::Coroutine => stable_mir::mir::CoroutineKind::Coroutine,
+            CoroutineKind::AsyncGen(_) => todo!(),
         }
     }
 }
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 5c1e703837a..7b9b7b85293 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -139,6 +139,9 @@ symbols! {
         AssertParamIsClone,
         AssertParamIsCopy,
         AssertParamIsEq,
+        AsyncGenFinished,
+        AsyncGenPending,
+        AsyncGenReady,
         AtomicBool,
         AtomicI128,
         AtomicI16,
@@ -423,6 +426,7 @@ symbols! {
         async_closure,
         async_fn_in_trait,
         async_fn_track_caller,
+        async_iterator,
         atomic,
         atomic_mod,
         atomics,
@@ -1200,6 +1204,7 @@ symbols! {
         pointer,
         pointer_like,
         poll,
+        poll_next,
         post_dash_lto: "post-lto",
         powerpc_target_feature,
         powf32,
diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
index 201fade5ad7..62d62bdfd11 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
@@ -207,6 +207,11 @@ pub(super) trait GoalKind<'tcx>:
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
+    fn consider_builtin_async_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
     /// A coroutine (that doesn't come from an `async` or `gen` desugaring) is known to
     /// implement `Coroutine<R, Yield = Y, Return = O>`, given the resume, yield,
     /// and return types of the coroutine computed during type-checking.
@@ -565,6 +570,8 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             G::consider_builtin_future_candidate(self, goal)
         } else if lang_items.iterator_trait() == Some(trait_def_id) {
             G::consider_builtin_iterator_candidate(self, goal)
+        } else if lang_items.async_iterator_trait() == Some(trait_def_id) {
+            G::consider_builtin_async_iterator_candidate(self, goal)
         } else if lang_items.coroutine_trait() == Some(trait_def_id) {
             G::consider_builtin_coroutine_candidate(self, goal)
         } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) {
diff --git a/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
index 867a520915f..2fe51b400ec 100644
--- a/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
@@ -510,6 +510,40 @@ impl<'tcx> assembly::GoalKind<'tcx> for NormalizesTo<'tcx> {
         )
     }
 
+    fn consider_builtin_async_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not AsyncIterators unless they come from `gen` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_async_gen(def_id) {
+            return Err(NoSolution);
+        }
+
+        ecx.probe_misc_candidate("builtin AsyncIterator kind").enter(|ecx| {
+            // Take `AsyncIterator<Item = I>` and turn it into the corresponding
+            // coroutine yield ty `Poll<Option<I>>`.
+            let expected_ty = Ty::new_adt(
+                tcx,
+                tcx.adt_def(tcx.require_lang_item(LangItem::Poll, None)),
+                tcx.mk_args(&[Ty::new_adt(
+                    tcx,
+                    tcx.adt_def(tcx.require_lang_item(LangItem::Option, None)),
+                    tcx.mk_args(&[goal.predicate.term.into()]),
+                )
+                .into()]),
+            );
+            let yield_ty = args.as_coroutine().yield_ty();
+            ecx.eq(goal.param_env, expected_ty, yield_ty)?;
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
+
     fn consider_builtin_coroutine_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
index 95712da3c5e..5807f7c6153 100644
--- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs
+++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
@@ -370,6 +370,30 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
+    fn consider_builtin_async_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        if goal.predicate.polarity != ty::ImplPolarity::Positive {
+            return Err(NoSolution);
+        }
+
+        let ty::Coroutine(def_id, _, _) = *goal.predicate.self_ty().kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not iterators unless they come from `gen` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_async_gen(def_id) {
+            return Err(NoSolution);
+        }
+
+        // Gen coroutines unconditionally implement `Iterator`
+        // Technically, we need to check that the iterator output type is Sized,
+        // but that's already proven by the coroutines being WF.
+        ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+    }
+
     fn consider_builtin_coroutine_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index 6b231a30ea7..7bf37cf7980 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -2587,6 +2587,23 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         CoroutineKind::Async(CoroutineSource::Closure) => {
                             format!("future created by async closure is not {trait_name}")
                         }
+                        CoroutineKind::AsyncGen(CoroutineSource::Fn) => self
+                            .tcx
+                            .parent(coroutine_did)
+                            .as_local()
+                            .map(|parent_did| self.tcx.local_def_id_to_hir_id(parent_did))
+                            .and_then(|parent_hir_id| hir.opt_name(parent_hir_id))
+                            .map(|name| {
+                                format!("async iterator returned by `{name}` is not {trait_name}")
+                            })?,
+                        CoroutineKind::AsyncGen(CoroutineSource::Block) => {
+                            format!("async iterator created by async gen block is not {trait_name}")
+                        }
+                        CoroutineKind::AsyncGen(CoroutineSource::Closure) => {
+                            format!(
+                                "async iterator created by async gen closure is not {trait_name}"
+                            )
+                        }
                         CoroutineKind::Gen(CoroutineSource::Fn) => self
                             .tcx
                             .parent(coroutine_did)
@@ -3127,7 +3144,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 let what = match self.tcx.coroutine_kind(coroutine_def_id) {
                     None
                     | Some(hir::CoroutineKind::Coroutine)
-                    | Some(hir::CoroutineKind::Gen(_)) => "yield",
+                    | Some(hir::CoroutineKind::Gen(_))
+                    // FIXME(gen_blocks): This could be yield or await...
+                    | Some(hir::CoroutineKind::AsyncGen(_)) => "yield",
                     Some(hir::CoroutineKind::Async(..)) => "await",
                 };
                 err.note(format!(
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index 1f94fbaf9f8..8fa0dceda87 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -1921,6 +1921,9 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             hir::CoroutineKind::Async(hir::CoroutineSource::Block) => "an async block",
             hir::CoroutineKind::Async(hir::CoroutineSource::Fn) => "an async function",
             hir::CoroutineKind::Async(hir::CoroutineSource::Closure) => "an async closure",
+            hir::CoroutineKind::AsyncGen(hir::CoroutineSource::Block) => "an async gen block",
+            hir::CoroutineKind::AsyncGen(hir::CoroutineSource::Fn) => "an async gen function",
+            hir::CoroutineKind::AsyncGen(hir::CoroutineSource::Closure) => "an async gen closure",
             hir::CoroutineKind::Gen(hir::CoroutineSource::Block) => "a gen block",
             hir::CoroutineKind::Gen(hir::CoroutineSource::Fn) => "a gen function",
             hir::CoroutineKind::Gen(hir::CoroutineSource::Closure) => "a gen closure",
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index 5b0829b5732..a08e35b566f 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -1823,11 +1823,18 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
                 let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
 
                 let lang_items = selcx.tcx().lang_items();
-                if [lang_items.coroutine_trait(), lang_items.future_trait(), lang_items.iterator_trait()].contains(&Some(trait_ref.def_id))
-                    || selcx.tcx().fn_trait_kind_from_def_id(trait_ref.def_id).is_some()
+                if [
+                    lang_items.coroutine_trait(),
+                    lang_items.future_trait(),
+                    lang_items.iterator_trait(),
+                    lang_items.async_iterator_trait(),
+                    lang_items.fn_trait(),
+                    lang_items.fn_mut_trait(),
+                    lang_items.fn_once_trait(),
+                ].contains(&Some(trait_ref.def_id))
                 {
                     true
-                } else if lang_items.discriminant_kind_trait() == Some(trait_ref.def_id) {
+                }else if lang_items.discriminant_kind_trait() == Some(trait_ref.def_id) {
                     match self_ty.kind() {
                         ty::Bool
                         | ty::Char
@@ -2042,6 +2049,8 @@ fn confirm_select_candidate<'cx, 'tcx>(
                 confirm_future_candidate(selcx, obligation, data)
             } else if lang_items.iterator_trait() == Some(trait_def_id) {
                 confirm_iterator_candidate(selcx, obligation, data)
+            } else if lang_items.async_iterator_trait() == Some(trait_def_id) {
+                confirm_async_iterator_candidate(selcx, obligation, data)
             } else if selcx.tcx().fn_trait_kind_from_def_id(trait_def_id).is_some() {
                 if obligation.predicate.self_ty().is_closure() {
                     confirm_closure_candidate(selcx, obligation, data)
@@ -2203,6 +2212,57 @@ fn confirm_iterator_candidate<'cx, 'tcx>(
         .with_addl_obligations(obligations)
 }
 
+fn confirm_async_iterator_candidate<'cx, 'tcx>(
+    selcx: &mut SelectionContext<'cx, 'tcx>,
+    obligation: &ProjectionTyObligation<'tcx>,
+    nested: Vec<PredicateObligation<'tcx>>,
+) -> Progress<'tcx> {
+    let ty::Coroutine(_, args, _) =
+        selcx.infcx.shallow_resolve(obligation.predicate.self_ty()).kind()
+    else {
+        unreachable!()
+    };
+    let gen_sig = args.as_coroutine().sig();
+    let Normalized { value: gen_sig, obligations } = normalize_with_depth(
+        selcx,
+        obligation.param_env,
+        obligation.cause.clone(),
+        obligation.recursion_depth + 1,
+        gen_sig,
+    );
+
+    debug!(?obligation, ?gen_sig, ?obligations, "confirm_async_iterator_candidate");
+
+    let tcx = selcx.tcx();
+    let iter_def_id = tcx.require_lang_item(LangItem::AsyncIterator, None);
+
+    let (trait_ref, yield_ty) = super::util::async_iterator_trait_ref_and_outputs(
+        tcx,
+        iter_def_id,
+        obligation.predicate.self_ty(),
+        gen_sig,
+    );
+
+    debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Item);
+
+    let ty::Adt(_poll_adt, args) = *yield_ty.kind() else {
+        bug!();
+    };
+    let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else {
+        bug!();
+    };
+    let item_ty = args.type_at(0);
+
+    let predicate = ty::ProjectionPredicate {
+        projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
+        term: item_ty.into(),
+    };
+
+    confirm_param_env_candidate(selcx, obligation, ty::Binder::dummy(predicate), false)
+        .with_addl_obligations(nested)
+        .with_addl_obligations(obligations)
+}
+
 fn confirm_builtin_candidate<'cx, 'tcx>(
     selcx: &mut SelectionContext<'cx, 'tcx>,
     obligation: &ProjectionTyObligation<'tcx>,
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index 367de517af2..c7d0ab71644 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -112,6 +112,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     self.assemble_future_candidates(obligation, &mut candidates);
                 } else if lang_items.iterator_trait() == Some(def_id) {
                     self.assemble_iterator_candidates(obligation, &mut candidates);
+                } else if lang_items.async_iterator_trait() == Some(def_id) {
+                    self.assemble_async_iterator_candidates(obligation, &mut candidates);
                 }
 
                 self.assemble_closure_candidates(obligation, &mut candidates);
@@ -258,6 +260,34 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         }
     }
 
+    fn assemble_async_iterator_candidates(
+        &mut self,
+        obligation: &PolyTraitObligation<'tcx>,
+        candidates: &mut SelectionCandidateSet<'tcx>,
+    ) {
+        let self_ty = obligation.self_ty().skip_binder();
+        if let ty::Coroutine(did, args, _) = *self_ty.kind() {
+            // gen constructs get lowered to a special kind of coroutine that
+            // should directly `impl AsyncIterator`.
+            if self.tcx().coroutine_is_async_gen(did) {
+                debug!(?self_ty, ?obligation, "assemble_iterator_candidates",);
+
+                // Can only confirm this candidate if we have constrained
+                // the `Yield` type to at least `Poll<Option<?0>>`..
+                let ty::Adt(_poll_def, args) = *args.as_coroutine().yield_ty().kind() else {
+                    candidates.ambiguous = true;
+                    return;
+                };
+                let ty::Adt(_option_def, _) = *args.type_at(0).kind() else {
+                    candidates.ambiguous = true;
+                    return;
+                };
+
+                candidates.vec.push(AsyncIteratorCandidate);
+            }
+        }
+    }
+
     /// Checks for the artificial impl that the compiler will create for an obligation like `X :
     /// FnMut<..>` where `X` is a closure type.
     ///
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 8567f4f0e70..4a342a7f6b1 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -98,6 +98,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ImplSource::Builtin(BuiltinImplSource::Misc, vtable_iterator)
             }
 
+            AsyncIteratorCandidate => {
+                let vtable_iterator = self.confirm_async_iterator_candidate(obligation)?;
+                ImplSource::Builtin(BuiltinImplSource::Misc, vtable_iterator)
+            }
+
             FnPointerCandidate { is_const } => {
                 let data = self.confirm_fn_pointer_candidate(obligation, is_const)?;
                 ImplSource::Builtin(BuiltinImplSource::Misc, data)
@@ -813,6 +818,35 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         Ok(nested)
     }
 
+    fn confirm_async_iterator_candidate(
+        &mut self,
+        obligation: &PolyTraitObligation<'tcx>,
+    ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
+        // Okay to skip binder because the args on coroutine types never
+        // touch bound regions, they just capture the in-scope
+        // type/region parameters.
+        let self_ty = self.infcx.shallow_resolve(obligation.self_ty().skip_binder());
+        let ty::Coroutine(coroutine_def_id, args, _) = *self_ty.kind() else {
+            bug!("closure candidate for non-closure {:?}", obligation);
+        };
+
+        debug!(?obligation, ?coroutine_def_id, ?args, "confirm_async_iterator_candidate");
+
+        let gen_sig = args.as_coroutine().sig();
+
+        let (trait_ref, _) = super::util::async_iterator_trait_ref_and_outputs(
+            self.tcx(),
+            obligation.predicate.def_id(),
+            obligation.predicate.no_bound_vars().expect("iterator has no bound vars").self_ty(),
+            gen_sig,
+        );
+
+        let nested = self.confirm_poly_trait_refs(obligation, ty::Binder::dummy(trait_ref))?;
+        debug!(?trait_ref, ?nested, "iterator candidate obligations");
+
+        Ok(nested)
+    }
+
     #[instrument(skip(self), level = "debug")]
     fn confirm_closure_candidate(
         &mut self,
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 6ba379467da..7f31a2529f5 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -1875,6 +1875,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1904,6 +1905,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1939,6 +1941,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1954,6 +1957,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -2061,6 +2065,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -2072,6 +2077,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
diff --git a/compiler/rustc_trait_selection/src/traits/util.rs b/compiler/rustc_trait_selection/src/traits/util.rs
index 5574badf238..98da3bc2fe9 100644
--- a/compiler/rustc_trait_selection/src/traits/util.rs
+++ b/compiler/rustc_trait_selection/src/traits/util.rs
@@ -308,6 +308,17 @@ pub fn iterator_trait_ref_and_outputs<'tcx>(
     (trait_ref, sig.yield_ty)
 }
 
+pub fn async_iterator_trait_ref_and_outputs<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    async_iterator_def_id: DefId,
+    self_ty: Ty<'tcx>,
+    sig: ty::GenSig<'tcx>,
+) -> (ty::TraitRef<'tcx>, Ty<'tcx>) {
+    assert!(!self_ty.has_escaping_bound_vars());
+    let trait_ref = ty::TraitRef::new(tcx, async_iterator_def_id, [self_ty]);
+    (trait_ref, sig.yield_ty)
+}
+
 pub fn impl_item_is_final(tcx: TyCtxt<'_>, assoc_item: &ty::AssocItem) -> bool {
     assoc_item.defaultness(tcx).is_final()
         && tcx.defaultness(assoc_item.container_id(tcx)).is_final()
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index a58e98ce99c..a5f11ca23e1 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -119,9 +119,9 @@ fn fn_sig_for_fn_abi<'tcx>(
                     // unlike for all other coroutine kinds.
                     env_ty
                 }
-                hir::CoroutineKind::Async(_) | hir::CoroutineKind::Coroutine => {
-                    Ty::new_adt(tcx, pin_adt_ref, pin_args)
-                }
+                hir::CoroutineKind::Async(_)
+                | hir::CoroutineKind::AsyncGen(_)
+                | hir::CoroutineKind::Coroutine => Ty::new_adt(tcx, pin_adt_ref, pin_args),
             };
 
             // The `FnSig` and the `ret_ty` here is for a coroutines main
@@ -168,6 +168,30 @@ fn fn_sig_for_fn_abi<'tcx>(
 
                     (None, ret_ty)
                 }
+                hir::CoroutineKind::AsyncGen(_) => {
+                    // The signature should be
+                    // `AsyncIterator::poll_next(_, &mut Context<'_>) -> Poll<Option<Output>>`
+                    assert_eq!(sig.return_ty, tcx.types.unit);
+
+                    // Yield type is already `Poll<Option<yield_ty>>`
+                    let ret_ty = sig.yield_ty;
+
+                    // We have to replace the `ResumeTy` that is used for type and borrow checking
+                    // with `&mut Context<'_>` which is used in codegen.
+                    #[cfg(debug_assertions)]
+                    {
+                        if let ty::Adt(resume_ty_adt, _) = sig.resume_ty.kind() {
+                            let expected_adt =
+                                tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
+                            assert_eq!(*resume_ty_adt, expected_adt);
+                        } else {
+                            panic!("expected `ResumeTy`, found `{:?}`", sig.resume_ty);
+                        };
+                    }
+                    let context_mut_ref = Ty::new_task_context(tcx);
+
+                    (Some(context_mut_ref), ret_ty)
+                }
                 hir::CoroutineKind::Coroutine => {
                     // The signature should be `Coroutine::resume(_, Resume) -> CoroutineState<Yield, Return>`
                     let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index a0f01d9eca9..f1c9bb23e5d 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -271,6 +271,21 @@ fn resolve_associated_item<'tcx>(
                     debug_assert!(tcx.defaultness(trait_item_id).has_value());
                     Some(Instance::new(trait_item_id, rcvr_args))
                 }
+            } else if Some(trait_ref.def_id) == lang_items.async_iterator_trait() {
+                let ty::Coroutine(coroutine_def_id, args, _) = *rcvr_args.type_at(0).kind() else {
+                    bug!()
+                };
+
+                if cfg!(debug_assertions) && tcx.item_name(trait_item_id) != sym::poll_next {
+                    span_bug!(
+                        tcx.def_span(coroutine_def_id),
+                        "no definition for `{trait_ref}::{}` for built-in coroutine type",
+                        tcx.item_name(trait_item_id)
+                    )
+                }
+
+                // `AsyncIterator::poll_next` is generated by the compiler.
+                Some(Instance { def: ty::InstanceDef::Item(coroutine_def_id), args })
             } else if Some(trait_ref.def_id) == lang_items.coroutine_trait() {
                 let ty::Coroutine(coroutine_def_id, args, _) = *rcvr_args.type_at(0).kind() else {
                     bug!()
diff --git a/library/core/src/async_iter/async_iter.rs b/library/core/src/async_iter/async_iter.rs
index 12a47f9fc76..8a45bd36f7a 100644
--- a/library/core/src/async_iter/async_iter.rs
+++ b/library/core/src/async_iter/async_iter.rs
@@ -13,6 +13,7 @@ use crate::task::{Context, Poll};
 #[unstable(feature = "async_iterator", issue = "79024")]
 #[must_use = "async iterators do nothing unless polled"]
 #[doc(alias = "Stream")]
+#[cfg_attr(not(bootstrap), lang = "async_iterator")]
 pub trait AsyncIterator {
     /// The type of items yielded by the async iterator.
     type Item;
@@ -109,3 +110,27 @@ where
         (**self).size_hint()
     }
 }
+
+#[unstable(feature = "async_gen_internals", issue = "none")]
+impl<T> Poll<Option<T>> {
+    /// A helper function for internal desugaring -- produces `Ready(Some(t))`,
+    /// which corresponds to the async iterator yielding a value.
+    #[unstable(feature = "async_gen_internals", issue = "none")]
+    #[cfg_attr(not(bootstrap), lang = "AsyncGenReady")]
+    pub fn async_gen_ready(t: T) -> Self {
+        Poll::Ready(Some(t))
+    }
+
+    /// A helper constant for internal desugaring -- produces `Pending`,
+    /// which corresponds to the async iterator pending on an `.await`.
+    #[unstable(feature = "async_gen_internals", issue = "none")]
+    #[cfg_attr(not(bootstrap), lang = "AsyncGenPending")]
+    // FIXME(gen_blocks): This probably could be deduplicated.
+    pub const PENDING: Self = Poll::Pending;
+
+    /// A helper constant for internal desugaring -- produces `Ready(None)`,
+    /// which corresponds to the async iterator finishing its iteration.
+    #[unstable(feature = "async_gen_internals", issue = "none")]
+    #[cfg_attr(not(bootstrap), lang = "AsyncGenFinished")]
+    pub const FINISHED: Self = Poll::Ready(None);
+}
-- 
cgit 1.4.1-3-g733a5


From 2806c2df7ba6eaaaf3e8ea82a10a3546977e070d Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Tue, 5 Dec 2023 21:39:36 +0000
Subject: coro_kind -> coroutine_kind

---
 compiler/rustc_ast/src/ast.rs                      | 15 ++++++---
 compiler/rustc_ast/src/mut_visit.rs                | 16 +++++-----
 compiler/rustc_ast/src/visit.rs                    |  2 +-
 compiler/rustc_ast_lowering/src/expr.rs            |  4 +--
 compiler/rustc_ast_lowering/src/item.rs            | 37 +++++++++++++---------
 compiler/rustc_ast_passes/src/ast_validation.rs    |  2 +-
 compiler/rustc_ast_pretty/src/pprust/state.rs      |  6 ++--
 compiler/rustc_ast_pretty/src/pprust/state/expr.rs |  4 +--
 compiler/rustc_builtin_macros/src/test.rs          |  4 +--
 compiler/rustc_expand/src/build.rs                 |  2 +-
 compiler/rustc_lint/src/early.rs                   |  4 +--
 compiler/rustc_parse/src/parser/expr.rs            |  2 +-
 compiler/rustc_parse/src/parser/item.rs            |  6 ++--
 compiler/rustc_parse/src/parser/ty.rs              |  4 +--
 compiler/rustc_resolve/src/def_collector.rs        |  4 +--
 compiler/rustc_resolve/src/late.rs                 | 14 +++++---
 .../clippy_lints/src/doc/needless_doctest_main.rs  |  2 +-
 src/tools/clippy/clippy_utils/src/ast_utils.rs     |  8 ++---
 src/tools/rustfmt/src/closures.rs                  | 21 ++++++++----
 src/tools/rustfmt/src/expr.rs                      |  2 +-
 src/tools/rustfmt/src/items.rs                     | 10 +++---
 src/tools/rustfmt/src/utils.rs                     |  4 +--
 tests/ui-fulldeps/pprust-expr-roundtrip.rs         |  2 +-
 23 files changed, 100 insertions(+), 75 deletions(-)

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 7cd1ff03400..4bb6586a693 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -1311,7 +1311,7 @@ pub struct Closure {
     pub binder: ClosureBinder,
     pub capture_clause: CaptureBy,
     pub constness: Const,
-    pub coro_kind: Option<CoroutineKind>,
+    pub coroutine_kind: Option<CoroutineKind>,
     pub movability: Movability,
     pub fn_decl: P<FnDecl>,
     pub body: P<Expr>,
@@ -2840,7 +2840,7 @@ pub struct FnHeader {
     /// The `unsafe` keyword, if any
     pub unsafety: Unsafe,
     /// Whether this is `async`, `gen`, or nothing.
-    pub coro_kind: Option<CoroutineKind>,
+    pub coroutine_kind: Option<CoroutineKind>,
     /// The `const` keyword, if any
     pub constness: Const,
     /// The `extern` keyword and corresponding ABI string, if any
@@ -2850,9 +2850,9 @@ pub struct FnHeader {
 impl FnHeader {
     /// Does this function header have any qualifiers or is it empty?
     pub fn has_qualifiers(&self) -> bool {
-        let Self { unsafety, coro_kind, constness, ext } = self;
+        let Self { unsafety, coroutine_kind, constness, ext } = self;
         matches!(unsafety, Unsafe::Yes(_))
-            || coro_kind.is_some()
+            || coroutine_kind.is_some()
             || matches!(constness, Const::Yes(_))
             || !matches!(ext, Extern::None)
     }
@@ -2860,7 +2860,12 @@ impl FnHeader {
 
 impl Default for FnHeader {
     fn default() -> FnHeader {
-        FnHeader { unsafety: Unsafe::No, coro_kind: None, constness: Const::No, ext: Extern::None }
+        FnHeader {
+            unsafety: Unsafe::No,
+            coroutine_kind: None,
+            constness: Const::No,
+            ext: Extern::None,
+        }
     }
 }
 
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index c6a31fbdbc3..64904736ff9 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -121,8 +121,8 @@ pub trait MutVisitor: Sized {
         noop_visit_fn_decl(d, self);
     }
 
-    fn visit_coro_kind(&mut self, a: &mut CoroutineKind) {
-        noop_visit_coro_kind(a, self);
+    fn visit_coroutine_kind(&mut self, a: &mut CoroutineKind) {
+        noop_visit_coroutine_kind(a, self);
     }
 
     fn visit_closure_binder(&mut self, b: &mut ClosureBinder) {
@@ -871,8 +871,8 @@ pub fn noop_visit_closure_binder<T: MutVisitor>(binder: &mut ClosureBinder, vis:
     }
 }
 
-pub fn noop_visit_coro_kind<T: MutVisitor>(coro_kind: &mut CoroutineKind, vis: &mut T) {
-    match coro_kind {
+pub fn noop_visit_coroutine_kind<T: MutVisitor>(coroutine_kind: &mut CoroutineKind, vis: &mut T) {
+    match coroutine_kind {
         CoroutineKind::Async { span, closure_id, return_impl_trait_id }
         | CoroutineKind::Gen { span, closure_id, return_impl_trait_id } => {
             vis.visit_span(span);
@@ -1171,9 +1171,9 @@ fn visit_const_item<T: MutVisitor>(
 }
 
 pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
-    let FnHeader { unsafety, coro_kind, constness, ext: _ } = header;
+    let FnHeader { unsafety, coroutine_kind, constness, ext: _ } = header;
     visit_constness(constness, vis);
-    coro_kind.as_mut().map(|coro_kind| vis.visit_coro_kind(coro_kind));
+    coroutine_kind.as_mut().map(|coroutine_kind| vis.visit_coroutine_kind(coroutine_kind));
     visit_unsafety(unsafety, vis);
 }
 
@@ -1407,7 +1407,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
             binder,
             capture_clause,
             constness,
-            coro_kind,
+            coroutine_kind,
             movability: _,
             fn_decl,
             body,
@@ -1416,7 +1416,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
         }) => {
             vis.visit_closure_binder(binder);
             visit_constness(constness, vis);
-            coro_kind.as_mut().map(|coro_kind| vis.visit_coro_kind(coro_kind));
+            coroutine_kind.as_mut().map(|coroutine_kind| vis.visit_coroutine_kind(coroutine_kind));
             vis.visit_capture_by(capture_clause);
             vis.visit_fn_decl(fn_decl);
             vis.visit_expr(body);
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index a303d6584f4..b3ce7e0ff06 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -861,7 +861,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
         ExprKind::Closure(box Closure {
             binder,
             capture_clause,
-            coro_kind: _,
+            coroutine_kind: _,
             constness: _,
             movability: _,
             fn_decl,
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index a8cbb42997d..b4100cd4eee 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -195,13 +195,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     binder,
                     capture_clause,
                     constness,
-                    coro_kind,
+                    coroutine_kind,
                     movability,
                     fn_decl,
                     body,
                     fn_decl_span,
                     fn_arg_span,
-                }) => match coro_kind {
+                }) => match coroutine_kind {
                     Some(
                         CoroutineKind::Async { closure_id, .. }
                         | CoroutineKind::Gen { closure_id, .. },
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 80854c8a6c0..ae50760c8f2 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -206,19 +206,25 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     // `impl Future<Output = T>` here because lower_body
                     // only cares about the input argument patterns in the function
                     // declaration (decl), not the return types.
-                    let coro_kind = header.coro_kind;
+                    let coroutine_kind = header.coroutine_kind;
                     let body_id = this.lower_maybe_coroutine_body(
                         span,
                         hir_id,
                         decl,
-                        coro_kind,
+                        coroutine_kind,
                         body.as_deref(),
                     );
 
                     let itctx = ImplTraitContext::Universal;
                     let (generics, decl) =
                         this.lower_generics(generics, header.constness, id, &itctx, |this| {
-                            this.lower_fn_decl(decl, id, *fn_sig_span, FnDeclKind::Fn, coro_kind)
+                            this.lower_fn_decl(
+                                decl,
+                                id,
+                                *fn_sig_span,
+                                FnDeclKind::Fn,
+                                coroutine_kind,
+                            )
                         });
                     let sig = hir::FnSig {
                         decl,
@@ -734,7 +740,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     sig,
                     i.id,
                     FnDeclKind::Trait,
-                    sig.header.coro_kind,
+                    sig.header.coroutine_kind,
                 );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Required(names)), false)
             }
@@ -743,7 +749,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     i.span,
                     hir_id,
                     &sig.decl,
-                    sig.header.coro_kind,
+                    sig.header.coroutine_kind,
                     Some(body),
                 );
                 let (generics, sig) = self.lower_method_sig(
@@ -751,7 +757,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     sig,
                     i.id,
                     FnDeclKind::Trait,
-                    sig.header.coro_kind,
+                    sig.header.coroutine_kind,
                 );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)), true)
             }
@@ -844,7 +850,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     i.span,
                     hir_id,
                     &sig.decl,
-                    sig.header.coro_kind,
+                    sig.header.coroutine_kind,
                     body.as_deref(),
                 );
                 let (generics, sig) = self.lower_method_sig(
@@ -852,7 +858,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     sig,
                     i.id,
                     if self.is_in_trait_impl { FnDeclKind::Impl } else { FnDeclKind::Inherent },
-                    sig.header.coro_kind,
+                    sig.header.coroutine_kind,
                 );
 
                 (generics, hir::ImplItemKind::Fn(sig, body_id))
@@ -1023,13 +1029,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         span: Span,
         fn_id: hir::HirId,
         decl: &FnDecl,
-        coro_kind: Option<CoroutineKind>,
+        coroutine_kind: Option<CoroutineKind>,
         body: Option<&Block>,
     ) -> hir::BodyId {
-        let (Some(coro_kind), Some(body)) = (coro_kind, body) else {
+        let (Some(coroutine_kind), Some(body)) = (coroutine_kind, body) else {
             return self.lower_fn_body_block(span, decl, body);
         };
-        let closure_id = match coro_kind {
+        let closure_id = match coroutine_kind {
             CoroutineKind::Async { closure_id, .. } | CoroutineKind::Gen { closure_id, .. } => {
                 closure_id
             }
@@ -1200,7 +1206,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
                 this.expr_block(body)
             };
-            let coroutine_expr = match coro_kind {
+            // FIXME(gen_blocks): Consider unifying the `make_*_expr` functions.
+            let coroutine_expr = match coroutine_kind {
                 CoroutineKind::Async { .. } => this.make_async_expr(
                     CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
                     closure_id,
@@ -1233,19 +1240,19 @@ impl<'hir> LoweringContext<'_, 'hir> {
         sig: &FnSig,
         id: NodeId,
         kind: FnDeclKind,
-        coro_kind: Option<CoroutineKind>,
+        coroutine_kind: Option<CoroutineKind>,
     ) -> (&'hir hir::Generics<'hir>, hir::FnSig<'hir>) {
         let header = self.lower_fn_header(sig.header);
         let itctx = ImplTraitContext::Universal;
         let (generics, decl) =
             self.lower_generics(generics, sig.header.constness, id, &itctx, |this| {
-                this.lower_fn_decl(&sig.decl, id, sig.span, kind, coro_kind)
+                this.lower_fn_decl(&sig.decl, id, sig.span, kind, coroutine_kind)
             });
         (generics, hir::FnSig { header, decl, span: self.lower_span(sig.span) })
     }
 
     fn lower_fn_header(&mut self, h: FnHeader) -> hir::FnHeader {
-        let asyncness = if let Some(CoroutineKind::Async { span, .. }) = h.coro_kind {
+        let asyncness = if let Some(CoroutineKind::Async { span, .. }) = h.coroutine_kind {
             hir::IsAsync::Async(span)
         } else {
             hir::IsAsync::NotAsync
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 554ed36b814..6811548fff3 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -1271,7 +1271,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         // Functions cannot both be `const async` or `const gen`
         if let Some(&FnHeader {
             constness: Const::Yes(cspan),
-            coro_kind:
+            coroutine_kind:
                 Some(
                     CoroutineKind::Async { span: aspan, .. }
                     | CoroutineKind::Gen { span: aspan, .. },
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 1ad28ffbf2b..01c8cb574bd 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -1490,8 +1490,8 @@ impl<'a> State<'a> {
         }
     }
 
-    fn print_coro_kind(&mut self, coro_kind: ast::CoroutineKind) {
-        match coro_kind {
+    fn print_coroutine_kind(&mut self, coroutine_kind: ast::CoroutineKind) {
+        match coroutine_kind {
             ast::CoroutineKind::Gen { .. } => {
                 self.word_nbsp("gen");
             }
@@ -1690,7 +1690,7 @@ impl<'a> State<'a> {
 
     fn print_fn_header_info(&mut self, header: ast::FnHeader) {
         self.print_constness(header.constness);
-        header.coro_kind.map(|coro_kind| self.print_coro_kind(coro_kind));
+        header.coroutine_kind.map(|coroutine_kind| self.print_coroutine_kind(coroutine_kind));
         self.print_unsafety(header.unsafety);
 
         match header.ext {
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index 0082d6e350e..2334c81243d 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -413,7 +413,7 @@ impl<'a> State<'a> {
                 binder,
                 capture_clause,
                 constness,
-                coro_kind,
+                coroutine_kind,
                 movability,
                 fn_decl,
                 body,
@@ -423,7 +423,7 @@ impl<'a> State<'a> {
                 self.print_closure_binder(binder);
                 self.print_constness(*constness);
                 self.print_movability(*movability);
-                coro_kind.map(|coro_kind| self.print_coro_kind(coro_kind));
+                coroutine_kind.map(|coroutine_kind| self.print_coroutine_kind(coroutine_kind));
                 self.print_capture_clause(*capture_clause);
 
                 self.print_fn_params_and_ret(fn_decl, true);
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index 81433155ecf..be4fd5dd36f 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -541,11 +541,11 @@ fn check_test_signature(
         return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "unsafe" }));
     }
 
-    if let Some(ast::CoroutineKind::Async { span, .. }) = f.sig.header.coro_kind {
+    if let Some(ast::CoroutineKind::Async { span, .. }) = f.sig.header.coroutine_kind {
         return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "async" }));
     }
 
-    if let Some(ast::CoroutineKind::Gen { span, .. }) = f.sig.header.coro_kind {
+    if let Some(ast::CoroutineKind::Gen { span, .. }) = f.sig.header.coroutine_kind {
         return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "gen" }));
     }
 
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index 794e11d87d1..49de6d9ae1e 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -547,7 +547,7 @@ impl<'a> ExtCtxt<'a> {
                 binder: ast::ClosureBinder::NotPresent,
                 capture_clause: ast::CaptureBy::Ref,
                 constness: ast::Const::No,
-                coro_kind: None,
+                coroutine_kind: None,
                 movability: ast::Movability::Movable,
                 fn_decl,
                 body,
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index 7c4f81a4c39..1673356ed1d 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -165,7 +165,7 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
             if let Some(
                 ast::CoroutineKind::Async { closure_id, .. }
                 | ast::CoroutineKind::Gen { closure_id, .. },
-            ) = sig.header.coro_kind
+            ) = sig.header.coroutine_kind
             {
                 self.check_id(closure_id);
             }
@@ -227,7 +227,7 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // it does not have a corresponding AST node
         match e.kind {
             ast::ExprKind::Closure(box ast::Closure {
-                coro_kind:
+                coroutine_kind:
                     Some(
                         ast::CoroutineKind::Async { closure_id, .. }
                         | ast::CoroutineKind::Gen { closure_id, .. },
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index b5a9e91ae94..7654ae7cd59 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -2285,7 +2285,7 @@ impl<'a> Parser<'a> {
                 binder,
                 capture_clause,
                 constness,
-                coro_kind: asyncness,
+                coroutine_kind: asyncness,
                 movability,
                 fn_decl,
                 body,
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 5fa9efb5af7..7b5271238a8 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -2561,7 +2561,7 @@ impl<'a> Parser<'a> {
                         return Ok(FnHeader {
                             constness: recover_constness,
                             unsafety: recover_unsafety,
-                            coro_kind: recover_asyncness,
+                            coroutine_kind: recover_asyncness,
                             ext,
                         });
                     }
@@ -2571,13 +2571,13 @@ impl<'a> Parser<'a> {
             }
         }
 
-        let coro_kind = match asyncness {
+        let coroutine_kind = match asyncness {
             Some(CoroutineKind::Async { .. }) => asyncness,
             Some(CoroutineKind::Gen { .. }) => unreachable!("asycness cannot be Gen"),
             None => genness,
         };
 
-        Ok(FnHeader { constness, unsafety, coro_kind, ext })
+        Ok(FnHeader { constness, unsafety, coroutine_kind, ext })
     }
 
     /// Parses the parameter list and result type of a function declaration.
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index f349140e8c3..da8cc05ff66 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -598,7 +598,7 @@ impl<'a> Parser<'a> {
             tokens: None,
         };
         let span_start = self.token.span;
-        let ast::FnHeader { ext, unsafety, constness, coro_kind } =
+        let ast::FnHeader { ext, unsafety, constness, coroutine_kind } =
             self.parse_fn_front_matter(&inherited_vis, Case::Sensitive)?;
         if self.may_recover() && self.token.kind == TokenKind::Lt {
             self.recover_fn_ptr_with_generics(lo, &mut params, param_insertion_point)?;
@@ -611,7 +611,7 @@ impl<'a> Parser<'a> {
             // cover it.
             self.sess.emit_err(FnPointerCannotBeConst { span: whole_span, qualifier: span });
         }
-        if let Some(ast::CoroutineKind::Async { span, .. }) = coro_kind {
+        if let Some(ast::CoroutineKind::Async { span, .. }) = coroutine_kind {
             self.sess.emit_err(FnPointerCannotBeAsync { span: whole_span, qualifier: span });
         }
         // FIXME(gen_blocks): emit a similar error for `gen fn()`
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index ab5d3b368eb..3bfdb5253c4 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -158,7 +158,7 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
         if let FnKind::Fn(_, _, sig, _, generics, body) = fn_kind {
             if let Some(
                 CoroutineKind::Async { closure_id, .. } | CoroutineKind::Gen { closure_id, .. },
-            ) = sig.header.coro_kind
+            ) = sig.header.coroutine_kind
             {
                 self.visit_generics(generics);
 
@@ -284,7 +284,7 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
                 // Async closures desugar to closures inside of closures, so
                 // we must create two defs.
                 let closure_def = self.create_def(expr.id, kw::Empty, DefKind::Closure, expr.span);
-                match closure.coro_kind {
+                match closure.coroutine_kind {
                     Some(
                         CoroutineKind::Async { closure_id, .. }
                         | CoroutineKind::Gen { closure_id, .. },
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index ad14f5e5225..09cc15a1d19 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -916,8 +916,10 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                             &sig.decl.output,
                         );
 
-                        if let Some((coro_node_id, _)) =
-                            sig.header.coro_kind.map(|coro_kind| coro_kind.return_id())
+                        if let Some((coro_node_id, _)) = sig
+                            .header
+                            .coroutine_kind
+                            .map(|coroutine_kind| coroutine_kind.return_id())
                         {
                             this.record_lifetime_params_for_impl_trait(coro_node_id);
                         }
@@ -942,8 +944,10 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                         this.visit_generics(generics);
 
                         let declaration = &sig.decl;
-                        let coro_node_id =
-                            sig.header.coro_kind.map(|coro_kind| coro_kind.return_id());
+                        let coro_node_id = sig
+                            .header
+                            .coroutine_kind
+                            .map(|coroutine_kind| coroutine_kind.return_id());
 
                         this.with_lifetime_rib(
                             LifetimeRibKind::AnonymousCreateParameter {
@@ -4294,7 +4298,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             //
             // Similarly, `gen |x| ...` gets desugared to `|x| gen {...}`, so we handle that too.
             ExprKind::Closure(box ast::Closure {
-                coro_kind: Some(_),
+                coroutine_kind: Some(_),
                 ref fn_decl,
                 ref body,
                 ..
diff --git a/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs b/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs
index 640d4a069ec..e019523e609 100644
--- a/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs
+++ b/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs
@@ -69,7 +69,7 @@ pub fn check(
                                 if !ignore {
                                     get_test_spans(&item, &mut test_attr_spans);
                                 }
-                                let is_async = matches!(sig.header.coro_kind, Some(CoroutineKind::Async { .. }));
+                                let is_async = matches!(sig.header.coroutine_kind, Some(CoroutineKind::Async { .. }));
                                 let returns_nothing = match &sig.decl.output {
                                     FnRetTy::Default(..) => true,
                                     FnRetTy::Ty(ty) if ty.kind.is_unit() => true,
diff --git a/src/tools/clippy/clippy_utils/src/ast_utils.rs b/src/tools/clippy/clippy_utils/src/ast_utils.rs
index 12403bbff3c..8373627d6a0 100644
--- a/src/tools/clippy/clippy_utils/src/ast_utils.rs
+++ b/src/tools/clippy/clippy_utils/src/ast_utils.rs
@@ -188,7 +188,7 @@ pub fn eq_expr(l: &Expr, r: &Expr) -> bool {
             Closure(box ast::Closure {
                 binder: lb,
                 capture_clause: lc,
-                coro_kind: la,
+                coroutine_kind: la,
                 movability: lm,
                 fn_decl: lf,
                 body: le,
@@ -197,7 +197,7 @@ pub fn eq_expr(l: &Expr, r: &Expr) -> bool {
             Closure(box ast::Closure {
                 binder: rb,
                 capture_clause: rc,
-                coro_kind: ra,
+                coroutine_kind: ra,
                 movability: rm,
                 fn_decl: rf,
                 body: re,
@@ -563,7 +563,7 @@ pub fn eq_fn_sig(l: &FnSig, r: &FnSig) -> bool {
     eq_fn_decl(&l.decl, &r.decl) && eq_fn_header(&l.header, &r.header)
 }
 
-fn eq_opt_coro_kind(l: Option<CoroutineKind>, r: Option<CoroutineKind>) -> bool {
+fn eq_opt_coroutine_kind(l: Option<CoroutineKind>, r: Option<CoroutineKind>) -> bool {
     match (l, r) {
         (Some(CoroutineKind::Async { .. }), Some(CoroutineKind::Async { .. }))
         | (Some(CoroutineKind::Gen { .. }), Some(CoroutineKind::Gen { .. })) => true,
@@ -574,7 +574,7 @@ fn eq_opt_coro_kind(l: Option<CoroutineKind>, r: Option<CoroutineKind>) -> bool
 
 pub fn eq_fn_header(l: &FnHeader, r: &FnHeader) -> bool {
     matches!(l.unsafety, Unsafe::No) == matches!(r.unsafety, Unsafe::No)
-        && eq_opt_coro_kind(l.coro_kind, r.coro_kind)
+        && eq_opt_coroutine_kind(l.coroutine_kind, r.coroutine_kind)
         && matches!(l.constness, Const::No) == matches!(r.constness, Const::No)
         && eq_ext(&l.ext, &r.ext)
 }
diff --git a/src/tools/rustfmt/src/closures.rs b/src/tools/rustfmt/src/closures.rs
index c1ce87eadcb..638955948e9 100644
--- a/src/tools/rustfmt/src/closures.rs
+++ b/src/tools/rustfmt/src/closures.rs
@@ -29,7 +29,7 @@ pub(crate) fn rewrite_closure(
     binder: &ast::ClosureBinder,
     constness: ast::Const,
     capture: ast::CaptureBy,
-    coro_kind: &Option<ast::CoroutineKind>,
+    coroutine_kind: &Option<ast::CoroutineKind>,
     movability: ast::Movability,
     fn_decl: &ast::FnDecl,
     body: &ast::Expr,
@@ -40,7 +40,16 @@ pub(crate) fn rewrite_closure(
     debug!("rewrite_closure {:?}", body);
 
     let (prefix, extra_offset) = rewrite_closure_fn_decl(
-        binder, constness, capture, coro_kind, movability, fn_decl, body, span, context, shape,
+        binder,
+        constness,
+        capture,
+        coroutine_kind,
+        movability,
+        fn_decl,
+        body,
+        span,
+        context,
+        shape,
     )?;
     // 1 = space between `|...|` and body.
     let body_shape = shape.offset_left(extra_offset)?;
@@ -233,7 +242,7 @@ fn rewrite_closure_fn_decl(
     binder: &ast::ClosureBinder,
     constness: ast::Const,
     capture: ast::CaptureBy,
-    coro_kind: &Option<ast::CoroutineKind>,
+    coroutine_kind: &Option<ast::CoroutineKind>,
     movability: ast::Movability,
     fn_decl: &ast::FnDecl,
     body: &ast::Expr,
@@ -263,7 +272,7 @@ fn rewrite_closure_fn_decl(
     } else {
         ""
     };
-    let coro = match coro_kind {
+    let coro = match coroutine_kind {
         Some(ast::CoroutineKind::Async { .. }) => "async ",
         Some(ast::CoroutineKind::Gen { .. }) => "gen ",
         None => "",
@@ -343,7 +352,7 @@ pub(crate) fn rewrite_last_closure(
             ref binder,
             constness,
             capture_clause,
-            ref coro_kind,
+            ref coroutine_kind,
             movability,
             ref fn_decl,
             ref body,
@@ -364,7 +373,7 @@ pub(crate) fn rewrite_last_closure(
             binder,
             constness,
             capture_clause,
-            coro_kind,
+            coroutine_kind,
             movability,
             fn_decl,
             body,
diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs
index 4515c27be37..a68bd6694ba 100644
--- a/src/tools/rustfmt/src/expr.rs
+++ b/src/tools/rustfmt/src/expr.rs
@@ -212,7 +212,7 @@ pub(crate) fn format_expr(
             &cl.binder,
             cl.constness,
             cl.capture_clause,
-            &cl.coro_kind,
+            &cl.coroutine_kind,
             cl.movability,
             &cl.fn_decl,
             &cl.body,
diff --git a/src/tools/rustfmt/src/items.rs b/src/tools/rustfmt/src/items.rs
index 4dff65f8cd0..a4256730f19 100644
--- a/src/tools/rustfmt/src/items.rs
+++ b/src/tools/rustfmt/src/items.rs
@@ -287,7 +287,7 @@ pub(crate) struct FnSig<'a> {
     decl: &'a ast::FnDecl,
     generics: &'a ast::Generics,
     ext: ast::Extern,
-    coro_kind: Cow<'a, Option<ast::CoroutineKind>>,
+    coroutine_kind: Cow<'a, Option<ast::CoroutineKind>>,
     constness: ast::Const,
     defaultness: ast::Defaultness,
     unsafety: ast::Unsafe,
@@ -302,7 +302,7 @@ impl<'a> FnSig<'a> {
     ) -> FnSig<'a> {
         FnSig {
             unsafety: method_sig.header.unsafety,
-            coro_kind: Cow::Borrowed(&method_sig.header.coro_kind),
+            coroutine_kind: Cow::Borrowed(&method_sig.header.coroutine_kind),
             constness: method_sig.header.constness,
             defaultness: ast::Defaultness::Final,
             ext: method_sig.header.ext,
@@ -328,7 +328,7 @@ impl<'a> FnSig<'a> {
                 generics,
                 ext: fn_sig.header.ext,
                 constness: fn_sig.header.constness,
-                coro_kind: Cow::Borrowed(&fn_sig.header.coro_kind),
+                coroutine_kind: Cow::Borrowed(&fn_sig.header.coroutine_kind),
                 defaultness,
                 unsafety: fn_sig.header.unsafety,
                 visibility: vis,
@@ -343,8 +343,8 @@ impl<'a> FnSig<'a> {
         result.push_str(&*format_visibility(context, self.visibility));
         result.push_str(format_defaultness(self.defaultness));
         result.push_str(format_constness(self.constness));
-        self.coro_kind
-            .map(|coro_kind| result.push_str(format_coro(&coro_kind)));
+        self.coroutine_kind
+            .map(|coroutine_kind| result.push_str(format_coro(&coroutine_kind)));
         result.push_str(format_unsafety(self.unsafety));
         result.push_str(&format_extern(
             self.ext,
diff --git a/src/tools/rustfmt/src/utils.rs b/src/tools/rustfmt/src/utils.rs
index 5805e417c04..4392763cea6 100644
--- a/src/tools/rustfmt/src/utils.rs
+++ b/src/tools/rustfmt/src/utils.rs
@@ -75,8 +75,8 @@ pub(crate) fn format_visibility(
 }
 
 #[inline]
-pub(crate) fn format_coro(coro_kind: &ast::CoroutineKind) -> &'static str {
-    match coro_kind {
+pub(crate) fn format_coro(coroutine_kind: &ast::CoroutineKind) -> &'static str {
+    match coroutine_kind {
         ast::CoroutineKind::Async { .. } => "async ",
         ast::CoroutineKind::Gen { .. } => "gen ",
     }
diff --git a/tests/ui-fulldeps/pprust-expr-roundtrip.rs b/tests/ui-fulldeps/pprust-expr-roundtrip.rs
index 9e581620ec1..fe5333643ed 100644
--- a/tests/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/tests/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -132,7 +132,7 @@ fn iter_exprs(depth: usize, f: &mut dyn FnMut(P<Expr>)) {
                         binder: ClosureBinder::NotPresent,
                         capture_clause: CaptureBy::Value { move_kw: DUMMY_SP },
                         constness: Const::No,
-                        coro_kind: None,
+                        coroutine_kind: None,
                         movability: Movability::Movable,
                         fn_decl: decl.clone(),
                         body: e,
-- 
cgit 1.4.1-3-g733a5


From a208bae00e1031ef3d7bb922c4f68cbbc362900c Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Tue, 5 Dec 2023 21:45:01 +0000
Subject: Support async gen fn

---
 compiler/rustc_ast/src/ast.rs                 | 11 +++++--
 compiler/rustc_ast/src/mut_visit.rs           |  3 +-
 compiler/rustc_ast_lowering/src/expr.rs       | 16 +++++----
 compiler/rustc_ast_lowering/src/item.rs       | 16 ++++++---
 compiler/rustc_ast_lowering/src/lib.rs        |  8 +++--
 compiler/rustc_ast_lowering/src/path.rs       |  2 +-
 compiler/rustc_ast_pretty/src/pprust/state.rs |  4 +++
 compiler/rustc_hir/src/hir.rs                 |  5 ---
 compiler/rustc_parse/messages.ftl             |  2 --
 compiler/rustc_parse/src/errors.rs            |  7 ----
 compiler/rustc_parse/src/parser/expr.rs       | 36 ++++++++++++--------
 compiler/rustc_parse/src/parser/item.rs       | 47 ++++++++++++---------------
 compiler/rustc_parse/src/parser/mod.rs        | 39 +++++++++++++---------
 compiler/rustc_resolve/src/def_collector.rs   |  3 +-
 tests/ui/coroutine/async_gen_fn.rs            | 12 ++++---
 tests/ui/coroutine/async_gen_fn.stderr        |  8 -----
 16 files changed, 115 insertions(+), 104 deletions(-)
 delete mode 100644 tests/ui/coroutine/async_gen_fn.stderr

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 4bb6586a693..c910fec381a 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -2415,10 +2415,12 @@ pub enum Unsafe {
 /// Iterator`.
 #[derive(Copy, Clone, Encodable, Decodable, Debug)]
 pub enum CoroutineKind {
-    /// `async`, which evaluates to `impl Future`
+    /// `async`, which returns an `impl Future`
     Async { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
-    /// `gen`, which evaluates to `impl Iterator`
+    /// `gen`, which returns an `impl Iterator`
     Gen { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
+    /// `async gen`, which returns an `impl AsyncIterator`
+    AsyncGen { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
 }
 
 impl CoroutineKind {
@@ -2435,7 +2437,10 @@ impl CoroutineKind {
     pub fn return_id(self) -> (NodeId, Span) {
         match self {
             CoroutineKind::Async { return_impl_trait_id, span, .. }
-            | CoroutineKind::Gen { return_impl_trait_id, span, .. } => (return_impl_trait_id, span),
+            | CoroutineKind::Gen { return_impl_trait_id, span, .. }
+            | CoroutineKind::AsyncGen { return_impl_trait_id, span, .. } => {
+                (return_impl_trait_id, span)
+            }
         }
     }
 }
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index 64904736ff9..904fc06e952 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -874,7 +874,8 @@ pub fn noop_visit_closure_binder<T: MutVisitor>(binder: &mut ClosureBinder, vis:
 pub fn noop_visit_coroutine_kind<T: MutVisitor>(coroutine_kind: &mut CoroutineKind, vis: &mut T) {
     match coroutine_kind {
         CoroutineKind::Async { span, closure_id, return_impl_trait_id }
-        | CoroutineKind::Gen { span, closure_id, return_impl_trait_id } => {
+        | CoroutineKind::Gen { span, closure_id, return_impl_trait_id }
+        | CoroutineKind::AsyncGen { span, closure_id, return_impl_trait_id } => {
             vis.visit_span(span);
             vis.visit_id(closure_id);
             vis.visit_id(return_impl_trait_id);
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index b4100cd4eee..dbf6ffd6d1c 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -13,6 +13,7 @@ use rustc_ast::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
+use rustc_middle::span_bug;
 use rustc_session::errors::report_lit_error;
 use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
@@ -202,15 +203,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     fn_decl_span,
                     fn_arg_span,
                 }) => match coroutine_kind {
-                    Some(
-                        CoroutineKind::Async { closure_id, .. }
-                        | CoroutineKind::Gen { closure_id, .. },
-                    ) => self.lower_expr_async_closure(
+                    Some(coroutine_kind) => self.lower_expr_coroutine_closure(
                         binder,
                         *capture_clause,
                         e.id,
                         hir_id,
-                        *closure_id,
+                        *coroutine_kind,
                         fn_decl,
                         body,
                         *fn_decl_span,
@@ -1098,18 +1096,22 @@ impl<'hir> LoweringContext<'_, 'hir> {
         (binder, params)
     }
 
-    fn lower_expr_async_closure(
+    fn lower_expr_coroutine_closure(
         &mut self,
         binder: &ClosureBinder,
         capture_clause: CaptureBy,
         closure_id: NodeId,
         closure_hir_id: hir::HirId,
-        inner_closure_id: NodeId,
+        coroutine_kind: CoroutineKind,
         decl: &FnDecl,
         body: &Expr,
         fn_decl_span: Span,
         fn_arg_span: Span,
     ) -> hir::ExprKind<'hir> {
+        let CoroutineKind::Async { closure_id: inner_closure_id, .. } = coroutine_kind else {
+            span_bug!(fn_decl_span, "`async gen` and `gen` closures are not supported, yet");
+        };
+
         if let &ClosureBinder::For { span, .. } = binder {
             self.tcx.sess.emit_err(NotSupportedForLifetimeBinderAsyncClosure { span });
         }
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index ae50760c8f2..577b8c1ff79 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -1035,11 +1035,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let (Some(coroutine_kind), Some(body)) = (coroutine_kind, body) else {
             return self.lower_fn_body_block(span, decl, body);
         };
-        let closure_id = match coroutine_kind {
-            CoroutineKind::Async { closure_id, .. } | CoroutineKind::Gen { closure_id, .. } => {
-                closure_id
-            }
-        };
+        let (CoroutineKind::Async { closure_id, .. }
+        | CoroutineKind::Gen { closure_id, .. }
+        | CoroutineKind::AsyncGen { closure_id, .. }) = coroutine_kind;
 
         self.lower_body(|this| {
             let mut parameters: Vec<hir::Param<'_>> = Vec::new();
@@ -1224,6 +1222,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     hir::CoroutineSource::Fn,
                     mkbody,
                 ),
+                CoroutineKind::AsyncGen { .. } => this.make_async_gen_expr(
+                    CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
+                    closure_id,
+                    None,
+                    body.span,
+                    hir::CoroutineSource::Fn,
+                    mkbody,
+                ),
             };
 
             let hir_id = this.lower_node_id(closure_id);
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 5dda8f5a6a3..c48024a3f4a 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -1904,7 +1904,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         let opaque_ty_node_id = match coro {
             CoroutineKind::Async { return_impl_trait_id, .. }
-            | CoroutineKind::Gen { return_impl_trait_id, .. } => return_impl_trait_id,
+            | CoroutineKind::Gen { return_impl_trait_id, .. }
+            | CoroutineKind::AsyncGen { return_impl_trait_id, .. } => return_impl_trait_id,
         };
 
         let captured_lifetimes: Vec<_> = self
@@ -1960,8 +1961,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         // "<$assoc_ty_name = T>"
         let (assoc_ty_name, trait_lang_item) = match coro {
-            CoroutineKind::Async { .. } => (hir::FN_OUTPUT_NAME, hir::LangItem::Future),
-            CoroutineKind::Gen { .. } => (hir::ITERATOR_ITEM_NAME, hir::LangItem::Iterator),
+            CoroutineKind::Async { .. } => (sym::Output, hir::LangItem::Future),
+            CoroutineKind::Gen { .. } => (sym::Item, hir::LangItem::Iterator),
+            CoroutineKind::AsyncGen { .. } => (sym::Item, hir::LangItem::AsyncIterator),
         };
 
         let future_args = self.arena.alloc(hir::GenericArgs {
diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs
index 7ab0805d086..efd80af5ef4 100644
--- a/compiler/rustc_ast_lowering/src/path.rs
+++ b/compiler/rustc_ast_lowering/src/path.rs
@@ -389,7 +389,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             FnRetTy::Default(_) => self.arena.alloc(self.ty_tup(*span, &[])),
         };
         let args = smallvec![GenericArg::Type(self.arena.alloc(self.ty_tup(*inputs_span, inputs)))];
-        let binding = self.assoc_ty_binding(hir::FN_OUTPUT_NAME, output_ty.span, output_ty);
+        let binding = self.assoc_ty_binding(sym::Output, output_ty.span, output_ty);
         (
             GenericArgsCtor {
                 args,
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 01c8cb574bd..ff36e6c2845 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -1498,6 +1498,10 @@ impl<'a> State<'a> {
             ast::CoroutineKind::Async { .. } => {
                 self.word_nbsp("async");
             }
+            ast::CoroutineKind::AsyncGen { .. } => {
+                self.word_nbsp("async");
+                self.word_nbsp("gen");
+            }
         }
     }
 
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index e2aef43dcf0..07128dc9205 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -2255,11 +2255,6 @@ pub enum ImplItemKind<'hir> {
     Type(&'hir Ty<'hir>),
 }
 
-/// The name of the associated type for `Fn` return types.
-pub const FN_OUTPUT_NAME: Symbol = sym::Output;
-/// The name of the associated type for `Iterator` item types.
-pub const ITERATOR_ITEM_NAME: Symbol = sym::Item;
-
 /// Bind a type to an associated type (i.e., `A = Foo`).
 ///
 /// Bindings like `A: Debug` are represented as a special type `A =
diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
index da51d9dbe9f..1c3c433d8b7 100644
--- a/compiler/rustc_parse/messages.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -23,8 +23,6 @@ parse_async_block_in_2015 = `async` blocks are only allowed in Rust 2018 or late
 parse_async_fn_in_2015 = `async fn` is not permitted in Rust 2015
     .label = to use `async fn`, switch to Rust 2018 or later
 
-parse_async_gen_fn = `async gen` functions are not supported
-
 parse_async_move_block_in_2015 = `async move` blocks are only allowed in Rust 2018 or later
 
 parse_async_move_order_incorrect = the order of `move` and `async` is incorrect
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 45f950db5c3..bc53ab83439 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -562,13 +562,6 @@ pub(crate) struct GenFn {
     pub span: Span,
 }
 
-#[derive(Diagnostic)]
-#[diag(parse_async_gen_fn)]
-pub(crate) struct AsyncGenFn {
-    #[primary_span]
-    pub span: Span,
-}
-
 #[derive(Diagnostic)]
 #[diag(parse_comma_after_base_struct)]
 #[note]
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 7654ae7cd59..c3bfece923d 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1442,21 +1442,21 @@ impl<'a> Parser<'a> {
             } else if this.token.uninterpolated_span().at_least_rust_2018() {
                 // `Span:.at_least_rust_2018()` is somewhat expensive; don't get it repeatedly.
                 if this.check_keyword(kw::Async) {
-                    if this.is_gen_block(kw::Async, 0) || this.is_gen_block(kw::Gen, 1) {
+                    // FIXME(gen_blocks): Parse `gen async` and suggest swap
+                    if this.is_gen_block(kw::Async, 0) {
                         // Check for `async {` and `async move {`,
                         // or `async gen {` and `async gen move {`.
                         this.parse_gen_block()
                     } else {
                         this.parse_expr_closure()
                     }
-                } else if this.eat_keyword(kw::Await) {
+                } else if this.token.uninterpolated_span().at_least_rust_2024()
+                    && (this.is_gen_block(kw::Gen, 0)
+                        || (this.check_keyword(kw::Async) && this.is_gen_block(kw::Gen, 1)))
+                {
+                    this.parse_gen_block()
+                } else if this.eat_keyword_noexpect(kw::Await) {
                     this.recover_incorrect_await_syntax(lo, this.prev_token.span)
-                } else if this.token.uninterpolated_span().at_least_rust_2024() {
-                    if this.is_gen_block(kw::Gen, 0) {
-                        this.parse_gen_block()
-                    } else {
-                        this.parse_expr_lit()
-                    }
                 } else {
                     this.parse_expr_lit()
                 }
@@ -2235,8 +2235,8 @@ impl<'a> Parser<'a> {
         let movability =
             if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable };
 
-        let asyncness = if self.token.uninterpolated_span().at_least_rust_2018() {
-            self.parse_asyncness(Case::Sensitive)
+        let coroutine_kind = if self.token.uninterpolated_span().at_least_rust_2018() {
+            self.parse_coroutine_kind(Case::Sensitive)
         } else {
             None
         };
@@ -2262,9 +2262,17 @@ impl<'a> Parser<'a> {
             }
         };
 
-        if let Some(CoroutineKind::Async { span, .. }) = asyncness {
-            // Feature-gate `async ||` closures.
-            self.sess.gated_spans.gate(sym::async_closure, span);
+        match coroutine_kind {
+            Some(CoroutineKind::Async { span, .. }) => {
+                // Feature-gate `async ||` closures.
+                self.sess.gated_spans.gate(sym::async_closure, span);
+            }
+            Some(CoroutineKind::Gen { span, .. }) | Some(CoroutineKind::AsyncGen { span, .. }) => {
+                // Feature-gate `gen ||` and `async gen ||` closures.
+                // FIXME(gen_blocks): This perhaps should be a different gate.
+                self.sess.gated_spans.gate(sym::gen_blocks, span);
+            }
+            None => {}
         }
 
         if self.token.kind == TokenKind::Semi
@@ -2285,7 +2293,7 @@ impl<'a> Parser<'a> {
                 binder,
                 capture_clause,
                 constness,
-                coroutine_kind: asyncness,
+                coroutine_kind,
                 movability,
                 fn_decl,
                 body,
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 7b5271238a8..d22cc04d182 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -2394,10 +2394,7 @@ impl<'a> Parser<'a> {
         let constness = self.parse_constness(case);
 
         let async_start_sp = self.token.span;
-        let asyncness = self.parse_asyncness(case);
-
-        let _gen_start_sp = self.token.span;
-        let genness = self.parse_genness(case);
+        let coroutine_kind = self.parse_coroutine_kind(case);
 
         let unsafe_start_sp = self.token.span;
         let unsafety = self.parse_unsafety(case);
@@ -2405,7 +2402,7 @@ impl<'a> Parser<'a> {
         let ext_start_sp = self.token.span;
         let ext = self.parse_extern(case);
 
-        if let Some(CoroutineKind::Async { span, .. }) = asyncness {
+        if let Some(CoroutineKind::Async { span, .. }) = coroutine_kind {
             if span.is_rust_2015() {
                 self.sess.emit_err(errors::AsyncFnIn2015 {
                     span,
@@ -2414,16 +2411,11 @@ impl<'a> Parser<'a> {
             }
         }
 
-        if let Some(CoroutineKind::Gen { span, .. }) = genness {
-            self.sess.gated_spans.gate(sym::gen_blocks, span);
-        }
-
-        if let (
-            Some(CoroutineKind::Async { span: async_span, .. }),
-            Some(CoroutineKind::Gen { span: gen_span, .. }),
-        ) = (asyncness, genness)
-        {
-            self.sess.emit_err(errors::AsyncGenFn { span: async_span.to(gen_span) });
+        match coroutine_kind {
+            Some(CoroutineKind::Gen { span, .. }) | Some(CoroutineKind::AsyncGen { span, .. }) => {
+                self.sess.gated_spans.gate(sym::gen_blocks, span);
+            }
+            Some(CoroutineKind::Async { .. }) | None => {}
         }
 
         if !self.eat_keyword_case(kw::Fn, case) {
@@ -2442,7 +2434,7 @@ impl<'a> Parser<'a> {
 
                     // We may be able to recover
                     let mut recover_constness = constness;
-                    let mut recover_asyncness = asyncness;
+                    let mut recover_coroutine_kind = coroutine_kind;
                     let mut recover_unsafety = unsafety;
                     // This will allow the machine fix to directly place the keyword in the correct place or to indicate
                     // that the keyword is already present and the second instance should be removed.
@@ -2455,15 +2447,24 @@ impl<'a> Parser<'a> {
                             }
                         }
                     } else if self.check_keyword(kw::Async) {
-                        match asyncness {
+                        match coroutine_kind {
                             Some(CoroutineKind::Async { span, .. }) => {
                                 Some(WrongKw::Duplicated(span))
                             }
+                            Some(CoroutineKind::AsyncGen { span, .. }) => {
+                                Some(WrongKw::Duplicated(span))
+                            }
                             Some(CoroutineKind::Gen { .. }) => {
-                                panic!("not sure how to recover here")
+                                recover_coroutine_kind = Some(CoroutineKind::AsyncGen {
+                                    span: self.token.span,
+                                    closure_id: DUMMY_NODE_ID,
+                                    return_impl_trait_id: DUMMY_NODE_ID,
+                                });
+                                // FIXME(gen_blocks): This span is wrong, didn't want to think about it.
+                                Some(WrongKw::Misplaced(unsafe_start_sp))
                             }
                             None => {
-                                recover_asyncness = Some(CoroutineKind::Async {
+                                recover_coroutine_kind = Some(CoroutineKind::Async {
                                     span: self.token.span,
                                     closure_id: DUMMY_NODE_ID,
                                     return_impl_trait_id: DUMMY_NODE_ID,
@@ -2561,7 +2562,7 @@ impl<'a> Parser<'a> {
                         return Ok(FnHeader {
                             constness: recover_constness,
                             unsafety: recover_unsafety,
-                            coroutine_kind: recover_asyncness,
+                            coroutine_kind: recover_coroutine_kind,
                             ext,
                         });
                     }
@@ -2571,12 +2572,6 @@ impl<'a> Parser<'a> {
             }
         }
 
-        let coroutine_kind = match asyncness {
-            Some(CoroutineKind::Async { .. }) => asyncness,
-            Some(CoroutineKind::Gen { .. }) => unreachable!("asycness cannot be Gen"),
-            None => genness,
-        };
-
         Ok(FnHeader { constness, unsafety, coroutine_kind, ext })
     }
 
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 2816386cbad..7a306823ed4 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -1125,23 +1125,30 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses asyncness: `async` or nothing.
-    fn parse_asyncness(&mut self, case: Case) -> Option<CoroutineKind> {
+    fn parse_coroutine_kind(&mut self, case: Case) -> Option<CoroutineKind> {
+        let span = self.token.uninterpolated_span();
         if self.eat_keyword_case(kw::Async, case) {
-            let span = self.prev_token.uninterpolated_span();
-            Some(CoroutineKind::Async {
-                span,
-                closure_id: DUMMY_NODE_ID,
-                return_impl_trait_id: DUMMY_NODE_ID,
-            })
-        } else {
-            None
-        }
-    }
-
-    /// Parses genness: `gen` or nothing.
-    fn parse_genness(&mut self, case: Case) -> Option<CoroutineKind> {
-        if self.token.span.at_least_rust_2024() && self.eat_keyword_case(kw::Gen, case) {
-            let span = self.prev_token.uninterpolated_span();
+            // FIXME(gen_blocks): Do we want to unconditionally parse `gen` and then
+            // error if edition <= 2024, like we do with async and edition <= 2018?
+            if self.token.uninterpolated_span().at_least_rust_2024()
+                && self.eat_keyword_case(kw::Gen, case)
+            {
+                let gen_span = self.prev_token.uninterpolated_span();
+                Some(CoroutineKind::AsyncGen {
+                    span: span.to(gen_span),
+                    closure_id: DUMMY_NODE_ID,
+                    return_impl_trait_id: DUMMY_NODE_ID,
+                })
+            } else {
+                Some(CoroutineKind::Async {
+                    span,
+                    closure_id: DUMMY_NODE_ID,
+                    return_impl_trait_id: DUMMY_NODE_ID,
+                })
+            }
+        } else if self.token.uninterpolated_span().at_least_rust_2024()
+            && self.eat_keyword_case(kw::Gen, case)
+        {
             Some(CoroutineKind::Gen {
                 span,
                 closure_id: DUMMY_NODE_ID,
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 3bfdb5253c4..4a62a2c4625 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -287,7 +287,8 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
                 match closure.coroutine_kind {
                     Some(
                         CoroutineKind::Async { closure_id, .. }
-                        | CoroutineKind::Gen { closure_id, .. },
+                        | CoroutineKind::Gen { closure_id, .. }
+                        | CoroutineKind::AsyncGen { closure_id, .. },
                     ) => self.create_def(closure_id, kw::Empty, DefKind::Closure, expr.span),
                     None => closure_def,
                 }
diff --git a/tests/ui/coroutine/async_gen_fn.rs b/tests/ui/coroutine/async_gen_fn.rs
index f8860e07f6c..f51fef43504 100644
--- a/tests/ui/coroutine/async_gen_fn.rs
+++ b/tests/ui/coroutine/async_gen_fn.rs
@@ -1,11 +1,13 @@
 // edition: 2024
 // compile-flags: -Zunstable-options
-#![feature(gen_blocks)]
+// check-pass
 
-// async generators are not yet supported, so this test makes sure they make some kind of reasonable
-// error.
+#![feature(gen_blocks, async_iterator)]
 
-async gen fn foo() {}
-//~^ `async gen` functions are not supported
+async fn bar() {}
+
+async gen fn foo() {
+    yield bar().await;
+}
 
 fn main() {}
diff --git a/tests/ui/coroutine/async_gen_fn.stderr b/tests/ui/coroutine/async_gen_fn.stderr
deleted file mode 100644
index 6857ebe6c79..00000000000
--- a/tests/ui/coroutine/async_gen_fn.stderr
+++ /dev/null
@@ -1,8 +0,0 @@
-error: `async gen` functions are not supported
-  --> $DIR/async_gen_fn.rs:8:1
-   |
-LL | async gen fn foo() {}
-   | ^^^^^^^^^
-
-error: aborting due to 1 previous error
-
-- 
cgit 1.4.1-3-g733a5


From 44911b7c67a0041d06ce959aaec5a3521155a09f Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Tue, 5 Dec 2023 21:53:18 +0000
Subject: Make some matches exhaustive to avoid bugs, fix tools

---
 compiler/rustc_ast_lowering/src/item.rs         |  1 +
 compiler/rustc_ast_passes/src/ast_validation.rs | 11 +++---
 compiler/rustc_builtin_macros/src/test.rs       | 30 ++++++++++++----
 compiler/rustc_lint/src/early.rs                | 23 ++++++------
 compiler/rustc_resolve/src/def_collector.rs     | 48 +++++++++++++------------
 src/tools/clippy/clippy_utils/src/ast_utils.rs  |  3 +-
 src/tools/rustfmt/src/closures.rs               |  1 +
 src/tools/rustfmt/src/utils.rs                  |  1 +
 8 files changed, 72 insertions(+), 46 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 577b8c1ff79..9d1f2684c39 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -1035,6 +1035,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let (Some(coroutine_kind), Some(body)) = (coroutine_kind, body) else {
             return self.lower_fn_body_block(span, decl, body);
         };
+        // FIXME(gen_blocks): Introduce `closure_id` method and remove ALL destructuring.
         let (CoroutineKind::Async { closure_id, .. }
         | CoroutineKind::Gen { closure_id, .. }
         | CoroutineKind::AsyncGen { closure_id, .. }) = coroutine_kind;
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 6811548fff3..0644c4cd6be 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -1271,14 +1271,15 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         // Functions cannot both be `const async` or `const gen`
         if let Some(&FnHeader {
             constness: Const::Yes(cspan),
-            coroutine_kind:
-                Some(
-                    CoroutineKind::Async { span: aspan, .. }
-                    | CoroutineKind::Gen { span: aspan, .. },
-                ),
+            coroutine_kind: Some(coro_kind),
             ..
         }) = fk.header()
         {
+            let aspan = match coro_kind {
+                CoroutineKind::Async { span: aspan, .. }
+                | CoroutineKind::Gen { span: aspan, .. }
+                | CoroutineKind::AsyncGen { span: aspan, .. } => aspan,
+            };
             // FIXME(gen_blocks): Report a different error for `const gen`
             self.err_handler().emit_err(errors::ConstAndAsync {
                 spans: vec![cspan, aspan],
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index be4fd5dd36f..794be25955d 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -541,12 +541,30 @@ fn check_test_signature(
         return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "unsafe" }));
     }
 
-    if let Some(ast::CoroutineKind::Async { span, .. }) = f.sig.header.coroutine_kind {
-        return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "async" }));
-    }
-
-    if let Some(ast::CoroutineKind::Gen { span, .. }) = f.sig.header.coroutine_kind {
-        return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "gen" }));
+    if let Some(coro_kind) = f.sig.header.coroutine_kind {
+        match coro_kind {
+            ast::CoroutineKind::Async { span, .. } => {
+                return Err(sd.emit_err(errors::TestBadFn {
+                    span: i.span,
+                    cause: span,
+                    kind: "async",
+                }));
+            }
+            ast::CoroutineKind::Gen { span, .. } => {
+                return Err(sd.emit_err(errors::TestBadFn {
+                    span: i.span,
+                    cause: span,
+                    kind: "gen",
+                }));
+            }
+            ast::CoroutineKind::AsyncGen { span, .. } => {
+                return Err(sd.emit_err(errors::TestBadFn {
+                    span: i.span,
+                    cause: span,
+                    kind: "async gen",
+                }));
+            }
+        }
     }
 
     // If the termination trait is active, the compiler will check that the output
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index 1673356ed1d..80c6feaa269 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -162,11 +162,10 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // Explicitly check for lints associated with 'closure_id', since
         // it does not have a corresponding AST node
         if let ast_visit::FnKind::Fn(_, _, sig, _, _, _) = fk {
-            if let Some(
-                ast::CoroutineKind::Async { closure_id, .. }
-                | ast::CoroutineKind::Gen { closure_id, .. },
-            ) = sig.header.coroutine_kind
-            {
+            if let Some(coro_kind) = sig.header.coroutine_kind {
+                let (ast::CoroutineKind::Async { closure_id, .. }
+                | ast::CoroutineKind::Gen { closure_id, .. }
+                | ast::CoroutineKind::AsyncGen { closure_id, .. }) = coro_kind;
                 self.check_id(closure_id);
             }
         }
@@ -227,13 +226,13 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // it does not have a corresponding AST node
         match e.kind {
             ast::ExprKind::Closure(box ast::Closure {
-                coroutine_kind:
-                    Some(
-                        ast::CoroutineKind::Async { closure_id, .. }
-                        | ast::CoroutineKind::Gen { closure_id, .. },
-                    ),
-                ..
-            }) => self.check_id(closure_id),
+                coroutine_kind: Some(coro_kind), ..
+            }) => {
+                let (ast::CoroutineKind::Async { closure_id, .. }
+                | ast::CoroutineKind::Gen { closure_id, .. }
+                | ast::CoroutineKind::AsyncGen { closure_id, .. }) = coro_kind;
+                self.check_id(closure_id);
+            }
             _ => {}
         }
         lint_callback!(self, check_expr_post, e);
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 4a62a2c4625..186dd28b142 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -156,29 +156,33 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
 
     fn visit_fn(&mut self, fn_kind: FnKind<'a>, span: Span, _: NodeId) {
         if let FnKind::Fn(_, _, sig, _, generics, body) = fn_kind {
-            if let Some(
-                CoroutineKind::Async { closure_id, .. } | CoroutineKind::Gen { closure_id, .. },
-            ) = sig.header.coroutine_kind
-            {
-                self.visit_generics(generics);
-
-                // For async functions, we need to create their inner defs inside of a
-                // closure to match their desugared representation. Besides that,
-                // we must mirror everything that `visit::walk_fn` below does.
-                self.visit_fn_header(&sig.header);
-                for param in &sig.decl.inputs {
-                    self.visit_param(param);
-                }
-                self.visit_fn_ret_ty(&sig.decl.output);
-                // If this async fn has no body (i.e. it's an async fn signature in a trait)
-                // then the closure_def will never be used, and we should avoid generating a
-                // def-id for it.
-                if let Some(body) = body {
-                    let closure_def =
-                        self.create_def(closure_id, kw::Empty, DefKind::Closure, span);
-                    self.with_parent(closure_def, |this| this.visit_block(body));
+            match sig.header.coroutine_kind {
+                Some(
+                    CoroutineKind::Async { closure_id, .. }
+                    | CoroutineKind::Gen { closure_id, .. }
+                    | CoroutineKind::AsyncGen { closure_id, .. },
+                ) => {
+                    self.visit_generics(generics);
+
+                    // For async functions, we need to create their inner defs inside of a
+                    // closure to match their desugared representation. Besides that,
+                    // we must mirror everything that `visit::walk_fn` below does.
+                    self.visit_fn_header(&sig.header);
+                    for param in &sig.decl.inputs {
+                        self.visit_param(param);
+                    }
+                    self.visit_fn_ret_ty(&sig.decl.output);
+                    // If this async fn has no body (i.e. it's an async fn signature in a trait)
+                    // then the closure_def will never be used, and we should avoid generating a
+                    // def-id for it.
+                    if let Some(body) = body {
+                        let closure_def =
+                            self.create_def(closure_id, kw::Empty, DefKind::Closure, span);
+                        self.with_parent(closure_def, |this| this.visit_block(body));
+                    }
+                    return;
                 }
-                return;
+                None => {}
             }
         }
 
diff --git a/src/tools/clippy/clippy_utils/src/ast_utils.rs b/src/tools/clippy/clippy_utils/src/ast_utils.rs
index 8373627d6a0..b4fe67023d8 100644
--- a/src/tools/clippy/clippy_utils/src/ast_utils.rs
+++ b/src/tools/clippy/clippy_utils/src/ast_utils.rs
@@ -566,7 +566,8 @@ pub fn eq_fn_sig(l: &FnSig, r: &FnSig) -> bool {
 fn eq_opt_coroutine_kind(l: Option<CoroutineKind>, r: Option<CoroutineKind>) -> bool {
     match (l, r) {
         (Some(CoroutineKind::Async { .. }), Some(CoroutineKind::Async { .. }))
-        | (Some(CoroutineKind::Gen { .. }), Some(CoroutineKind::Gen { .. })) => true,
+        | (Some(CoroutineKind::Gen { .. }), Some(CoroutineKind::Gen { .. }))
+        | (Some(CoroutineKind::AsyncGen { .. }), Some(CoroutineKind::AsyncGen { .. })) => true,
         (None, None) => true,
         _ => false,
     }
diff --git a/src/tools/rustfmt/src/closures.rs b/src/tools/rustfmt/src/closures.rs
index 638955948e9..f698f494ae5 100644
--- a/src/tools/rustfmt/src/closures.rs
+++ b/src/tools/rustfmt/src/closures.rs
@@ -275,6 +275,7 @@ fn rewrite_closure_fn_decl(
     let coro = match coroutine_kind {
         Some(ast::CoroutineKind::Async { .. }) => "async ",
         Some(ast::CoroutineKind::Gen { .. }) => "gen ",
+        Some(ast::CoroutineKind::AsyncGen { .. }) => "async gen ",
         None => "",
     };
     let mover = if matches!(capture, ast::CaptureBy::Value { .. }) {
diff --git a/src/tools/rustfmt/src/utils.rs b/src/tools/rustfmt/src/utils.rs
index 4392763cea6..7d7bbf11529 100644
--- a/src/tools/rustfmt/src/utils.rs
+++ b/src/tools/rustfmt/src/utils.rs
@@ -79,6 +79,7 @@ pub(crate) fn format_coro(coroutine_kind: &ast::CoroutineKind) -> &'static str {
     match coroutine_kind {
         ast::CoroutineKind::Async { .. } => "async ",
         ast::CoroutineKind::Gen { .. } => "gen ",
+        ast::CoroutineKind::AsyncGen { .. } => "async gen ",
     }
 }
 
-- 
cgit 1.4.1-3-g733a5


From 4c770585622ea6159b78af6b8323722c6636c33c Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Thu, 7 Dec 2023 18:36:55 +0000
Subject: HACK: constrain yield type in check_fn so that projection is
 successful even with no yield

---
 compiler/rustc_hir_typeck/src/check.rs | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs
index 91c96d6f76f..19b566ff9fa 100644
--- a/compiler/rustc_hir_typeck/src/check.rs
+++ b/compiler/rustc_hir_typeck/src/check.rs
@@ -59,9 +59,7 @@ pub(super) fn check_fn<'a, 'tcx>(
         && can_be_coroutine.is_some()
     {
         let yield_ty = match kind {
-            hir::CoroutineKind::Gen(..)
-            | hir::CoroutineKind::AsyncGen(..)
-            | hir::CoroutineKind::Coroutine => {
+            hir::CoroutineKind::Gen(..) | hir::CoroutineKind::Coroutine => {
                 let yield_ty = fcx.next_ty_var(TypeVariableOrigin {
                     kind: TypeVariableOriginKind::TypeInference,
                     span,
@@ -69,6 +67,28 @@ pub(super) fn check_fn<'a, 'tcx>(
                 fcx.require_type_is_sized(yield_ty, span, traits::SizedYieldType);
                 yield_ty
             }
+            // HACK(-Ztrait-solver=next): In the *old* trait solver, we must eagerly
+            // guide inference on the yield type so that we can handle `AsyncIterator`
+            // in this block in projection correctly. In the new trait solver, it is
+            // not a problem.
+            hir::CoroutineKind::AsyncGen(..) => {
+                let yield_ty = fcx.next_ty_var(TypeVariableOrigin {
+                    kind: TypeVariableOriginKind::TypeInference,
+                    span,
+                });
+                fcx.require_type_is_sized(yield_ty, span, traits::SizedYieldType);
+
+                Ty::new_adt(
+                    tcx,
+                    tcx.adt_def(tcx.require_lang_item(hir::LangItem::Poll, Some(span))),
+                    tcx.mk_args(&[Ty::new_adt(
+                        tcx,
+                        tcx.adt_def(tcx.require_lang_item(hir::LangItem::Option, Some(span))),
+                        tcx.mk_args(&[yield_ty.into()]),
+                    )
+                    .into()]),
+                )
+            }
             hir::CoroutineKind::Async(..) => Ty::new_unit(tcx),
         };
 
-- 
cgit 1.4.1-3-g733a5


From 11375c86571ce58646e84cf47df884a3bc2a9934 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Thu, 7 Dec 2023 17:39:02 +0000
Subject: Add tests

---
 compiler/rustc_ast_lowering/src/lib.rs       | 25 +++++---
 tests/ui/coroutine/async_gen_fn.e2024.stderr | 12 ++++
 tests/ui/coroutine/async_gen_fn.none.stderr  | 18 ++++++
 tests/ui/coroutine/async_gen_fn.rs           | 16 ++---
 tests/ui/coroutine/async_gen_fn_iter.rs      | 96 ++++++++++++++++++++++++++++
 5 files changed, 148 insertions(+), 19 deletions(-)
 create mode 100644 tests/ui/coroutine/async_gen_fn.e2024.stderr
 create mode 100644 tests/ui/coroutine/async_gen_fn.none.stderr
 create mode 100644 tests/ui/coroutine/async_gen_fn_iter.rs

diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index c48024a3f4a..753650f7324 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -132,6 +132,7 @@ struct LoweringContext<'a, 'hir> {
 
     allow_try_trait: Lrc<[Symbol]>,
     allow_gen_future: Lrc<[Symbol]>,
+    allow_async_iterator: Lrc<[Symbol]>,
 
     /// Mapping from generics `def_id`s to TAIT generics `def_id`s.
     /// For each captured lifetime (e.g., 'a), we create a new lifetime parameter that is a generic
@@ -176,6 +177,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             } else {
                 [sym::gen_future].into()
             },
+            // FIXME(gen_blocks): how does `closure_track_caller`
+            allow_async_iterator: [sym::gen_future, sym::async_iterator].into(),
             generics_def_id_map: Default::default(),
             host_param_id: None,
         }
@@ -1900,14 +1903,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         fn_span: Span,
     ) -> hir::FnRetTy<'hir> {
         let span = self.lower_span(fn_span);
-        let opaque_ty_span = self.mark_span_with_reason(DesugaringKind::Async, span, None);
 
-        let opaque_ty_node_id = match coro {
-            CoroutineKind::Async { return_impl_trait_id, .. }
-            | CoroutineKind::Gen { return_impl_trait_id, .. }
-            | CoroutineKind::AsyncGen { return_impl_trait_id, .. } => return_impl_trait_id,
+        let (opaque_ty_node_id, allowed_features) = match coro {
+            CoroutineKind::Async { return_impl_trait_id, .. } => (return_impl_trait_id, None),
+            CoroutineKind::Gen { return_impl_trait_id, .. } => (return_impl_trait_id, None),
+            CoroutineKind::AsyncGen { return_impl_trait_id, .. } => {
+                (return_impl_trait_id, Some(self.allow_async_iterator.clone()))
+            }
         };
 
+        let opaque_ty_span =
+            self.mark_span_with_reason(DesugaringKind::Async, span, allowed_features);
+
         let captured_lifetimes: Vec<_> = self
             .resolver
             .take_extra_lifetime_params(opaque_ty_node_id)
@@ -1926,7 +1933,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 let bound = this.lower_coroutine_fn_output_type_to_bound(
                     output,
                     coro,
-                    span,
+                    opaque_ty_span,
                     ImplTraitContext::ReturnPositionOpaqueTy {
                         origin: hir::OpaqueTyOrigin::FnReturn(fn_def_id),
                         fn_kind,
@@ -1945,7 +1952,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         &mut self,
         output: &FnRetTy,
         coro: CoroutineKind,
-        span: Span,
+        opaque_ty_span: Span,
         nested_impl_trait_context: ImplTraitContext,
     ) -> hir::GenericBound<'hir> {
         // Compute the `T` in `Future<Output = T>` from the return type.
@@ -1968,14 +1975,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         let future_args = self.arena.alloc(hir::GenericArgs {
             args: &[],
-            bindings: arena_vec![self; self.assoc_ty_binding(assoc_ty_name, span, output_ty)],
+            bindings: arena_vec![self; self.assoc_ty_binding(assoc_ty_name, opaque_ty_span, output_ty)],
             parenthesized: hir::GenericArgsParentheses::No,
             span_ext: DUMMY_SP,
         });
 
         hir::GenericBound::LangItemTrait(
             trait_lang_item,
-            self.lower_span(span),
+            opaque_ty_span,
             self.next_id(),
             future_args,
         )
diff --git a/tests/ui/coroutine/async_gen_fn.e2024.stderr b/tests/ui/coroutine/async_gen_fn.e2024.stderr
new file mode 100644
index 00000000000..d24cdbbc30d
--- /dev/null
+++ b/tests/ui/coroutine/async_gen_fn.e2024.stderr
@@ -0,0 +1,12 @@
+error[E0658]: gen blocks are experimental
+  --> $DIR/async_gen_fn.rs:4:1
+   |
+LL | async gen fn foo() {}
+   | ^^^^^^^^^
+   |
+   = note: see issue #117078 <https://github.com/rust-lang/rust/issues/117078> for more information
+   = help: add `#![feature(gen_blocks)]` to the crate attributes to enable
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/coroutine/async_gen_fn.none.stderr b/tests/ui/coroutine/async_gen_fn.none.stderr
new file mode 100644
index 00000000000..7950251a75d
--- /dev/null
+++ b/tests/ui/coroutine/async_gen_fn.none.stderr
@@ -0,0 +1,18 @@
+error[E0670]: `async fn` is not permitted in Rust 2015
+  --> $DIR/async_gen_fn.rs:4:1
+   |
+LL | async gen fn foo() {}
+   | ^^^^^ to use `async fn`, switch to Rust 2018 or later
+   |
+   = help: pass `--edition 2021` to `rustc`
+   = note: for more on editions, read https://doc.rust-lang.org/edition-guide
+
+error: expected one of `extern`, `fn`, or `unsafe`, found `gen`
+  --> $DIR/async_gen_fn.rs:4:7
+   |
+LL | async gen fn foo() {}
+   |       ^^^ expected one of `extern`, `fn`, or `unsafe`
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0670`.
diff --git a/tests/ui/coroutine/async_gen_fn.rs b/tests/ui/coroutine/async_gen_fn.rs
index f51fef43504..20564106f99 100644
--- a/tests/ui/coroutine/async_gen_fn.rs
+++ b/tests/ui/coroutine/async_gen_fn.rs
@@ -1,13 +1,9 @@
-// edition: 2024
-// compile-flags: -Zunstable-options
-// check-pass
+// revisions: e2024 none
+//[e2024] compile-flags: --edition 2024 -Zunstable-options
 
-#![feature(gen_blocks, async_iterator)]
-
-async fn bar() {}
-
-async gen fn foo() {
-    yield bar().await;
-}
+async gen fn foo() {}
+//[none]~^ ERROR: `async fn` is not permitted in Rust 2015
+//[none]~| ERROR: expected one of `extern`, `fn`, or `unsafe`, found `gen`
+//[e2024]~^^^ ERROR: gen blocks are experimental
 
 fn main() {}
diff --git a/tests/ui/coroutine/async_gen_fn_iter.rs b/tests/ui/coroutine/async_gen_fn_iter.rs
new file mode 100644
index 00000000000..6f8f3feb87e
--- /dev/null
+++ b/tests/ui/coroutine/async_gen_fn_iter.rs
@@ -0,0 +1,96 @@
+// edition: 2024
+// compile-flags: -Zunstable-options
+// run-pass
+
+#![feature(gen_blocks, async_iterator)]
+
+// make sure that a ridiculously simple async gen fn works as an iterator.
+
+async fn pause() {
+    // this doesn't actually do anything, lol
+}
+
+async fn one() -> i32 {
+    1
+}
+
+async fn two() -> i32 {
+    2
+}
+
+async gen fn foo() -> i32 {
+    yield one().await;
+    pause().await;
+    yield two().await;
+    pause().await;
+    yield 3;
+    pause().await;
+}
+
+async fn async_main() {
+    let mut iter = std::pin::pin!(foo());
+    assert_eq!(iter.next().await, Some(1));
+    assert_eq!(iter.as_mut().next().await, Some(2));
+    assert_eq!(iter.as_mut().next().await, Some(3));
+    assert_eq!(iter.as_mut().next().await, None);
+}
+
+// ------------------------------------------------------------------------- //
+// Implementation Details Below...
+
+use std::pin::Pin;
+use std::task::*;
+use std::async_iter::AsyncIterator;
+use std::future::Future;
+
+trait AsyncIterExt {
+    fn next(&mut self) -> Next<'_, Self>;
+}
+
+impl<T> AsyncIterExt for T {
+    fn next(&mut self) -> Next<'_, Self> {
+        Next { s: self }
+    }
+}
+
+struct Next<'s, S: ?Sized> {
+    s: &'s mut S,
+}
+
+impl<'s, S: AsyncIterator> Future for Next<'s, S> where S: Unpin {
+    type Output = Option<S::Item>;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<S::Item>> {
+        Pin::new(&mut *self.s).poll_next(cx)
+    }
+}
+
+pub fn noop_waker() -> Waker {
+    let raw = RawWaker::new(std::ptr::null(), &NOOP_WAKER_VTABLE);
+
+    // SAFETY: the contracts for RawWaker and RawWakerVTable are upheld
+    unsafe { Waker::from_raw(raw) }
+}
+
+const NOOP_WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new(noop_clone, noop, noop, noop);
+
+unsafe fn noop_clone(_p: *const ()) -> RawWaker {
+    RawWaker::new(std::ptr::null(), &NOOP_WAKER_VTABLE)
+}
+
+unsafe fn noop(_p: *const ()) {}
+
+fn main() {
+    let mut fut = async_main();
+
+    // Poll loop, just to test the future...
+    let waker = noop_waker();
+    let ctx = &mut Context::from_waker(&waker);
+
+    loop {
+        match unsafe { Pin::new_unchecked(&mut fut).poll(ctx) } {
+            Poll::Pending => {}
+            Poll::Ready(()) => break,
+        }
+    }
+}
-- 
cgit 1.4.1-3-g733a5


From cb41509601fc4059dd0c712034efa577a7ef46e3 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Wed, 22 Nov 2023 23:44:58 +0000
Subject: Uplift canonicalizer into new trait solver crate

---
 Cargo.lock                                         |   8 +
 compiler/rustc_infer/src/infer/mod.rs              |  72 ++--
 .../src/infer/region_constraints/mod.rs            |   5 +
 compiler/rustc_middle/src/ty/consts.rs             |  16 +-
 compiler/rustc_middle/src/ty/context.rs            |  36 ++
 compiler/rustc_middle/src/ty/mod.rs                |  32 +-
 compiler/rustc_middle/src/ty/sty.rs                |  10 +-
 compiler/rustc_next_trait_solver/Cargo.toml        |   7 +
 .../rustc_next_trait_solver/src/canonicalizer.rs   | 425 ++++++++++++++++++++
 compiler/rustc_next_trait_solver/src/lib.rs        |   1 +
 compiler/rustc_trait_selection/Cargo.toml          |   1 +
 .../src/solve/canonicalize.rs                      | 428 ---------------------
 .../src/solve/eval_ctxt/canonical.rs               |   2 +-
 compiler/rustc_trait_selection/src/solve/mod.rs    |   1 -
 compiler/rustc_type_ir/src/canonical.rs            |  13 +-
 compiler/rustc_type_ir/src/const_kind.rs           |  16 +-
 compiler/rustc_type_ir/src/debug.rs                |  53 ++-
 compiler/rustc_type_ir/src/infcx.rs                |  40 ++
 compiler/rustc_type_ir/src/interner.rs             |  52 ++-
 compiler/rustc_type_ir/src/lib.rs                  |  10 +-
 compiler/rustc_type_ir/src/ty_kind.rs              |  16 +-
 21 files changed, 739 insertions(+), 505 deletions(-)
 create mode 100644 compiler/rustc_next_trait_solver/Cargo.toml
 create mode 100644 compiler/rustc_next_trait_solver/src/canonicalizer.rs
 create mode 100644 compiler/rustc_next_trait_solver/src/lib.rs
 delete mode 100644 compiler/rustc_trait_selection/src/solve/canonicalize.rs
 create mode 100644 compiler/rustc_type_ir/src/infcx.rs

diff --git a/Cargo.lock b/Cargo.lock
index 1d7f93c789a..03a075b345c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4303,6 +4303,13 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "rustc_next_trait_solver"
+version = "0.0.0"
+dependencies = [
+ "rustc_type_ir",
+]
+
 [[package]]
 name = "rustc_parse"
 version = "0.0.0"
@@ -4571,6 +4578,7 @@ dependencies = [
  "rustc_infer",
  "rustc_macros",
  "rustc_middle",
+ "rustc_next_trait_solver",
  "rustc_parse_format",
  "rustc_query_system",
  "rustc_session",
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index 32c09e491c7..3a71251e73d 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -345,37 +345,61 @@ pub struct InferCtxt<'tcx> {
 impl<'tcx> ty::InferCtxtLike for InferCtxt<'tcx> {
     type Interner = TyCtxt<'tcx>;
 
-    fn universe_of_ty(&self, ty: ty::InferTy) -> Option<ty::UniverseIndex> {
-        use InferTy::*;
-        match ty {
-            // FIXME(BoxyUwU): this is kind of jank and means that printing unresolved
-            // ty infers will give you the universe of the var it resolved to not the universe
-            // it actually had. It also means that if you have a `?0.1` and infer it to `u8` then
-            // try to print out `?0.1` it will just print `?0`.
-            TyVar(ty_vid) => match self.probe_ty_var(ty_vid) {
-                Err(universe) => Some(universe),
-                Ok(_) => None,
-            },
-            IntVar(_) | FloatVar(_) | FreshTy(_) | FreshIntTy(_) | FreshFloatTy(_) => None,
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn universe_of_ty(&self, vid: TyVid) -> Option<ty::UniverseIndex> {
+        // FIXME(BoxyUwU): this is kind of jank and means that printing unresolved
+        // ty infers will give you the universe of the var it resolved to not the universe
+        // it actually had. It also means that if you have a `?0.1` and infer it to `u8` then
+        // try to print out `?0.1` it will just print `?0`.
+        match self.probe_ty_var(vid) {
+            Err(universe) => Some(universe),
+            Ok(_) => None,
         }
     }
 
-    fn universe_of_ct(&self, ct: ty::InferConst) -> Option<ty::UniverseIndex> {
-        use ty::InferConst::*;
-        match ct {
-            // Same issue as with `universe_of_ty`
-            Var(ct_vid) => match self.probe_const_var(ct_vid) {
-                Err(universe) => Some(universe),
-                Ok(_) => None,
-            },
-            EffectVar(_) => None,
-            Fresh(_) => None,
+    fn universe_of_ct(&self, ct: ConstVid) -> Option<ty::UniverseIndex> {
+        // Same issue as with `universe_of_ty`
+        match self.probe_const_var(ct) {
+            Err(universe) => Some(universe),
+            Ok(_) => None,
         }
     }
 
     fn universe_of_lt(&self, lt: ty::RegionVid) -> Option<ty::UniverseIndex> {
         Some(self.universe_of_region_vid(lt))
     }
+
+    fn root_ty_var(&self, vid: TyVid) -> TyVid {
+        self.root_var(vid)
+    }
+
+    fn probe_ty_var(&self, vid: TyVid) -> Option<Ty<'tcx>> {
+        self.probe_ty_var(vid).ok()
+    }
+
+    fn root_lt_var(&self, vid: ty::RegionVid) -> ty::RegionVid {
+        self.root_region_var(vid)
+    }
+
+    fn probe_lt_var(&self, vid: ty::RegionVid) -> Option<ty::Region<'tcx>> {
+        let re = self
+            .inner
+            .borrow_mut()
+            .unwrap_region_constraints()
+            .opportunistic_resolve_var(self.tcx, vid);
+        if re.is_var() { None } else { Some(re) }
+    }
+
+    fn root_ct_var(&self, vid: ConstVid) -> ConstVid {
+        self.root_const_var(vid)
+    }
+
+    fn probe_ct_var(&self, vid: ConstVid) -> Option<ty::Const<'tcx>> {
+        self.probe_const_var(vid).ok()
+    }
 }
 
 /// See the `error_reporting` module for more details.
@@ -1347,6 +1371,10 @@ impl<'tcx> InferCtxt<'tcx> {
         self.inner.borrow_mut().type_variables().root_var(var)
     }
 
+    pub fn root_region_var(&self, var: ty::RegionVid) -> ty::RegionVid {
+        self.inner.borrow_mut().unwrap_region_constraints().root_var(var)
+    }
+
     pub fn root_const_var(&self, var: ty::ConstVid) -> ty::ConstVid {
         self.inner.borrow_mut().const_unification_table().find(var).vid
     }
diff --git a/compiler/rustc_infer/src/infer/region_constraints/mod.rs b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
index cbd8040c9f1..5c043b1d3dd 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/mod.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
@@ -623,6 +623,11 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         }
     }
 
+    pub fn root_var(&mut self, vid: ty::RegionVid) -> ty::RegionVid {
+        let mut ut = self.unification_table_mut(); // FIXME(rust-lang/ena#42): unnecessary mut
+        ut.find(vid).vid
+    }
+
     fn combine_map(&mut self, t: CombineMapType) -> &mut CombineMap<'tcx> {
         match t {
             Glb => &mut self.glbs,
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index e48840fac20..78cc2f91796 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -7,7 +7,7 @@ use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::LocalDefId;
 use rustc_macros::HashStable;
-use rustc_type_ir::{TypeFlags, WithCachedTypeInfo};
+use rustc_type_ir::{ConstTy, IntoKind, TypeFlags, WithCachedTypeInfo};
 
 mod int;
 mod kind;
@@ -26,6 +26,20 @@ use super::sty::ConstKind;
 #[rustc_pass_by_value]
 pub struct Const<'tcx>(pub(super) Interned<'tcx, WithCachedTypeInfo<ConstData<'tcx>>>);
 
+impl<'tcx> IntoKind for Const<'tcx> {
+    type Kind = ConstKind<'tcx>;
+
+    fn kind(&self) -> ConstKind<'tcx> {
+        (*self).kind().clone()
+    }
+}
+
+impl<'tcx> ConstTy<TyCtxt<'tcx>> for Const<'tcx> {
+    fn ty(&self) -> Ty<'tcx> {
+        (*self).ty()
+    }
+}
+
 /// Typed constant value.
 #[derive(PartialEq, Eq, PartialOrd, Ord, Hash, HashStable, TyEncodable, TyDecodable)]
 pub struct ConstData<'tcx> {
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 3012434ad3f..a39acc9859f 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -131,6 +131,42 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     ) -> (Self::Ty, ty::Mutability) {
         (ty, mutbl)
     }
+
+    fn mk_canonical_var_infos(
+        &self,
+        infos: &[rustc_type_ir::CanonicalVarInfo<Self>],
+    ) -> Self::CanonicalVars {
+        (*self).mk_canonical_var_infos(infos)
+    }
+
+    fn mk_bound_ty(
+        &self,
+        debruijn: rustc_type_ir::DebruijnIndex,
+        var: rustc_type_ir::BoundVar,
+    ) -> Self::Ty {
+        Ty::new_bound(*self, debruijn, ty::BoundTy { var, kind: ty::BoundTyKind::Anon })
+    }
+
+    fn mk_bound_region(
+        &self,
+        debruijn: rustc_type_ir::DebruijnIndex,
+        var: rustc_type_ir::BoundVar,
+    ) -> Self::Region {
+        Region::new_bound(
+            *self,
+            debruijn,
+            ty::BoundRegion { var, kind: ty::BoundRegionKind::BrAnon },
+        )
+    }
+
+    fn mk_bound_const(
+        &self,
+        debruijn: rustc_type_ir::DebruijnIndex,
+        var: rustc_type_ir::BoundVar,
+        ty: Self::Ty,
+    ) -> Self::Const {
+        Const::new_bound(*self, debruijn, var, ty)
+    }
 }
 
 type InternedSet<'tcx, T> = ShardedHashMap<InternedInSet<'tcx, T>, ()>;
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 203c9eb65df..6aaa363eb01 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -474,6 +474,14 @@ pub struct CReaderCacheKey {
 #[rustc_pass_by_value]
 pub struct Ty<'tcx>(Interned<'tcx, WithCachedTypeInfo<TyKind<'tcx>>>);
 
+impl<'tcx> IntoKind for Ty<'tcx> {
+    type Kind = TyKind<'tcx>;
+
+    fn kind(&self) -> TyKind<'tcx> {
+        (*self).kind().clone()
+    }
+}
+
 impl EarlyParamRegion {
     /// Does this early bound region have a name? Early bound regions normally
     /// always have names except when using anonymous lifetimes (`'_`).
@@ -1554,8 +1562,12 @@ impl rustc_type_ir::Placeholder for PlaceholderRegion {
         self.bound.var
     }
 
-    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
-        Placeholder { universe: ui, ..self }
+    fn with_updated_universe(&self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..*self }
+    }
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self {
+        Placeholder { universe: ui, bound: BoundRegion { var, kind: BoundRegionKind::BrAnon } }
     }
 }
 
@@ -1570,8 +1582,12 @@ impl rustc_type_ir::Placeholder for PlaceholderType {
         self.bound.var
     }
 
-    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
-        Placeholder { universe: ui, ..self }
+    fn with_updated_universe(&self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..*self }
+    }
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self {
+        Placeholder { universe: ui, bound: BoundTy { var, kind: BoundTyKind::Anon } }
     }
 }
 
@@ -1593,8 +1609,12 @@ impl rustc_type_ir::Placeholder for PlaceholderConst {
         self.bound
     }
 
-    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
-        Placeholder { universe: ui, ..self }
+    fn with_updated_universe(&self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..*self }
+    }
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self {
+        Placeholder { universe: ui, bound: var }
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index e221b4e8bec..ed2f8f0a6ab 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -6,7 +6,7 @@ use crate::infer::canonical::Canonical;
 use crate::ty::visit::ValidateBoundVars;
 use crate::ty::InferTy::*;
 use crate::ty::{
-    self, AdtDef, Discr, Term, Ty, TyCtxt, TypeFlags, TypeSuperVisitable, TypeVisitable,
+    self, AdtDef, Discr, IntoKind, Term, Ty, TyCtxt, TypeFlags, TypeSuperVisitable, TypeVisitable,
     TypeVisitableExt, TypeVisitor,
 };
 use crate::ty::{GenericArg, GenericArgs, GenericArgsRef};
@@ -1477,6 +1477,14 @@ impl ParamConst {
 #[rustc_pass_by_value]
 pub struct Region<'tcx>(pub Interned<'tcx, RegionKind<'tcx>>);
 
+impl<'tcx> IntoKind for Region<'tcx> {
+    type Kind = RegionKind<'tcx>;
+
+    fn kind(&self) -> RegionKind<'tcx> {
+        **self
+    }
+}
+
 impl<'tcx> Region<'tcx> {
     #[inline]
     pub fn new_early_param(
diff --git a/compiler/rustc_next_trait_solver/Cargo.toml b/compiler/rustc_next_trait_solver/Cargo.toml
new file mode 100644
index 00000000000..88c4b62c54c
--- /dev/null
+++ b/compiler/rustc_next_trait_solver/Cargo.toml
@@ -0,0 +1,7 @@
+[package]
+name = "rustc_next_trait_solver"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+rustc_type_ir = { path = "../rustc_type_ir" }
diff --git a/compiler/rustc_next_trait_solver/src/canonicalizer.rs b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
new file mode 100644
index 00000000000..c2e6e82f2af
--- /dev/null
+++ b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
@@ -0,0 +1,425 @@
+use std::cmp::Ordering;
+
+use rustc_type_ir::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
+use rustc_type_ir::{
+    BoundVar, Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, ConstKind,
+    ConstTy, DebruijnIndex, InferConst, InferCtxtLike, InferTy, Interner, IntoKind, Placeholder,
+    RegionKind, TyKind, UniverseIndex, INNERMOST,
+};
+
+/// Whether we're canonicalizing a query input or the query response.
+///
+/// When canonicalizing an input we're in the context of the caller
+/// while canonicalizing the response happens in the context of the
+/// query.
+#[derive(Debug, Clone, Copy)]
+pub enum CanonicalizeMode {
+    Input,
+    /// FIXME: We currently return region constraints referring to
+    /// placeholders and inference variables from a binder instantiated
+    /// inside of the query.
+    ///
+    /// In the long term we should eagerly deal with these constraints
+    /// inside of the query and only propagate constraints which are
+    /// actually nameable by the caller.
+    Response {
+        /// The highest universe nameable by the caller.
+        ///
+        /// All variables in a universe nameable by the caller get mapped
+        /// to the root universe in the response and then mapped back to
+        /// their correct universe when applying the query response in the
+        /// context of the caller.
+        ///
+        /// This doesn't work for universes created inside of the query so
+        /// we do remember their universe in the response.
+        max_input_universe: UniverseIndex,
+    },
+}
+
+pub struct Canonicalizer<'a, Infcx: InferCtxtLike> {
+    infcx: &'a Infcx,
+    canonicalize_mode: CanonicalizeMode,
+
+    variables: &'a mut Vec<<Infcx::Interner as Interner>::GenericArg>,
+    primitive_var_infos: Vec<CanonicalVarInfo<Infcx::Interner>>,
+    binder_index: DebruijnIndex,
+}
+
+impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infcx> {
+    pub fn canonicalize<T: TypeFoldable<I>>(
+        infcx: &'a Infcx,
+        canonicalize_mode: CanonicalizeMode,
+        variables: &'a mut Vec<<I as Interner>::GenericArg>,
+        value: T,
+    ) -> Canonical<I, T> {
+        let mut canonicalizer = Canonicalizer {
+            infcx,
+            canonicalize_mode,
+
+            variables,
+            primitive_var_infos: Vec::new(),
+            binder_index: INNERMOST,
+        };
+
+        let value = value.fold_with(&mut canonicalizer);
+        // FIXME: Restore these assertions. Should we uplift type flags?
+        // assert!(!value.has_infer(), "unexpected infer in {value:?}");
+        // assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
+
+        let (max_universe, variables) = canonicalizer.finalize();
+
+        Canonical { max_universe, variables, value }
+    }
+
+    fn finalize(self) -> (UniverseIndex, <I as Interner>::CanonicalVars) {
+        let mut var_infos = self.primitive_var_infos;
+        // See the rustc-dev-guide section about how we deal with universes
+        // during canonicalization in the new solver.
+        match self.canonicalize_mode {
+            // We try to deduplicate as many query calls as possible and hide
+            // all information which should not matter for the solver.
+            //
+            // For this we compress universes as much as possible.
+            CanonicalizeMode::Input => {}
+            // When canonicalizing a response we map a universes already entered
+            // by the caller to the root universe and only return useful universe
+            // information for placeholders and inference variables created inside
+            // of the query.
+            CanonicalizeMode::Response { max_input_universe } => {
+                for var in var_infos.iter_mut() {
+                    let uv = var.universe();
+                    let new_uv =
+                        UniverseIndex::from(uv.index().saturating_sub(max_input_universe.index()));
+                    *var = var.with_updated_universe(new_uv);
+                }
+                let max_universe = var_infos
+                    .iter()
+                    .map(|info| info.universe())
+                    .max()
+                    .unwrap_or(UniverseIndex::ROOT);
+
+                let var_infos = self.infcx.interner().mk_canonical_var_infos(&var_infos);
+                return (max_universe, var_infos);
+            }
+        }
+
+        // Given a `var_infos` with existentials `En` and universals `Un` in
+        // universes `n`, this algorithm compresses them in place so that:
+        //
+        // - the new universe indices are as small as possible
+        // - we only create a new universe if we would otherwise put a placeholder in
+        //   the same compressed universe as an existential which cannot name it
+        //
+        // Let's walk through an example:
+        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 0
+        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 1
+        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 1, next_orig_uv: 2
+        // - var_infos: [E0, U1, E5, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 5
+        // - var_infos: [E0, U1, E1, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 6
+        // - var_infos: [E0, U1, E1, U1, E1, E2, U2], curr_compressed_uv: 2, next_orig_uv: -
+        //
+        // This algorithm runs in `O(n²)` where `n` is the number of different universe
+        // indices in the input. This should be fine as `n` is expected to be small.
+        let mut curr_compressed_uv = UniverseIndex::ROOT;
+        let mut existential_in_new_uv = false;
+        let mut next_orig_uv = Some(UniverseIndex::ROOT);
+        while let Some(orig_uv) = next_orig_uv.take() {
+            let mut update_uv = |var: &mut CanonicalVarInfo<I>, orig_uv, is_existential| {
+                let uv = var.universe();
+                match uv.cmp(&orig_uv) {
+                    Ordering::Less => (), // Already updated
+                    Ordering::Equal => {
+                        if is_existential {
+                            existential_in_new_uv = true;
+                        } else if existential_in_new_uv {
+                            //  `var` is a placeholder from a universe which is not nameable
+                            // by an existential which we already put into the compressed
+                            // universe `curr_compressed_uv`. We therefore have to create a
+                            // new universe for `var`.
+                            curr_compressed_uv = curr_compressed_uv.next_universe();
+                            existential_in_new_uv = false;
+                        }
+
+                        *var = var.with_updated_universe(curr_compressed_uv);
+                    }
+                    Ordering::Greater => {
+                        // We can ignore this variable in this iteration. We only look at
+                        // universes which actually occur in the input for performance.
+                        //
+                        // For this we set `next_orig_uv` to the next smallest, not yet compressed,
+                        // universe of the input.
+                        if next_orig_uv.map_or(true, |curr_next_uv| uv.cannot_name(curr_next_uv)) {
+                            next_orig_uv = Some(uv);
+                        }
+                    }
+                }
+            };
+
+            // For each universe which occurs in the input, we first iterate over all
+            // placeholders and then over all inference variables.
+            //
+            // Whenever we compress the universe of a placeholder, no existential with
+            // an already compressed universe can name that placeholder.
+            for is_existential in [false, true] {
+                for var in var_infos.iter_mut() {
+                    // We simply put all regions from the input into the highest
+                    // compressed universe, so we only deal with them at the end.
+                    if !var.is_region() {
+                        if is_existential == var.is_existential() {
+                            update_uv(var, orig_uv, is_existential)
+                        }
+                    }
+                }
+            }
+        }
+
+        for var in var_infos.iter_mut() {
+            if var.is_region() {
+                assert!(var.is_existential());
+                let compressed_var = var.with_updated_universe(curr_compressed_uv);
+                *var = compressed_var;
+            }
+        }
+
+        let var_infos = self.infcx.interner().mk_canonical_var_infos(&var_infos);
+        (curr_compressed_uv, var_infos)
+    }
+}
+
+impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonicalizer<'_, Infcx> {
+    fn interner(&self) -> I {
+        self.infcx.interner()
+    }
+
+    fn fold_binder<T>(&mut self, t: I::Binder<T>) -> I::Binder<T>
+    where
+        T: TypeFoldable<I>,
+        I::Binder<T>: TypeSuperFoldable<I>,
+    {
+        self.binder_index.shift_in(1);
+        let t = t.super_fold_with(self);
+        self.binder_index.shift_out(1);
+        t
+    }
+
+    fn fold_region(&mut self, r: I::Region) -> I::Region {
+        let kind = match r.kind() {
+            RegionKind::ReBound(..) => return r,
+
+            // We may encounter `ReStatic` in item signatures or the hidden type
+            // of an opaque. `ReErased` should only be encountered in the hidden
+            // type of an opaque for regions that are ignored for the purposes of
+            // captures.
+            //
+            // FIXME: We should investigate the perf implications of not uniquifying
+            // `ReErased`. We may be able to short-circuit registering region
+            // obligations if we encounter a `ReErased` on one side, for example.
+            RegionKind::ReStatic | RegionKind::ReErased => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
+                CanonicalizeMode::Response { .. } => return r,
+            },
+
+            RegionKind::ReEarlyParam(_) | RegionKind::ReLateParam(_) => {
+                match self.canonicalize_mode {
+                    CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
+                    CanonicalizeMode::Response { .. } => {
+                        panic!("unexpected region in response: {r:?}")
+                    }
+                }
+            }
+
+            RegionKind::RePlaceholder(placeholder) => match self.canonicalize_mode {
+                // We canonicalize placeholder regions as existentials in query inputs.
+                CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
+                CanonicalizeMode::Response { max_input_universe } => {
+                    // If we have a placeholder region inside of a query, it must be from
+                    // a new universe.
+                    if max_input_universe.can_name(placeholder.universe()) {
+                        panic!("new placeholder in universe {max_input_universe:?}: {r:?}");
+                    }
+                    CanonicalVarKind::PlaceholderRegion(placeholder)
+                }
+            },
+
+            RegionKind::ReVar(vid) => {
+                assert_eq!(
+                    self.infcx.root_lt_var(vid.clone()),
+                    vid,
+                    "region vid should have been resolved fully before canonicalization"
+                );
+                assert_eq!(
+                    self.infcx.probe_lt_var(vid.clone()),
+                    None,
+                    "region vid should have been resolved fully before canonicalization"
+                );
+
+                match self.canonicalize_mode {
+                    CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
+                    CanonicalizeMode::Response { .. } => {
+                        CanonicalVarKind::Region(self.infcx.universe_of_lt(vid).unwrap())
+                    }
+                }
+            }
+            RegionKind::ReError(_) => return r,
+        };
+
+        let existing_bound_var = match self.canonicalize_mode {
+            CanonicalizeMode::Input => None,
+            CanonicalizeMode::Response { .. } => {
+                let r = r.clone().into();
+                self.variables.iter().position(|v| v == &r).map(BoundVar::from)
+            }
+        };
+
+        let var = existing_bound_var.unwrap_or_else(|| {
+            let var = BoundVar::from(self.variables.len());
+            self.variables.push(r.into());
+            self.primitive_var_infos.push(CanonicalVarInfo { kind });
+            var
+        });
+
+        self.interner().mk_bound_region(self.binder_index, var)
+    }
+
+    fn fold_ty(&mut self, t: I::Ty) -> I::Ty
+    where
+        I::Ty: TypeSuperFoldable<I>,
+    {
+        let kind = match t.kind() {
+            TyKind::Infer(i) => match i {
+                InferTy::TyVar(vid) => {
+                    assert_eq!(
+                        self.infcx.root_ty_var(vid),
+                        vid,
+                        "ty vid should have been resolved fully before canonicalization"
+                    );
+                    assert_eq!(
+                        self.infcx.probe_ty_var(vid),
+                        None,
+                        "ty vid should have been resolved fully before canonicalization"
+                    );
+
+                    CanonicalVarKind::Ty(CanonicalTyVarKind::General(
+                        self.infcx
+                            .universe_of_ty(vid)
+                            .unwrap_or_else(|| panic!("ty var should have been resolved: {t:?}")),
+                    ))
+                }
+                InferTy::IntVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Int),
+                InferTy::FloatVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Float),
+                InferTy::FreshTy(_) | InferTy::FreshIntTy(_) | InferTy::FreshFloatTy(_) => {
+                    todo!()
+                }
+            },
+            TyKind::Placeholder(placeholder) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(Placeholder::new(
+                    placeholder.universe(),
+                    self.variables.len().into(),
+                )),
+                CanonicalizeMode::Response { .. } => CanonicalVarKind::PlaceholderTy(placeholder),
+            },
+            TyKind::Param(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(Placeholder::new(
+                    UniverseIndex::ROOT,
+                    self.variables.len().into(),
+                )),
+                CanonicalizeMode::Response { .. } => panic!("param ty in response: {t:?}"),
+            },
+            TyKind::Bool
+            | TyKind::Char
+            | TyKind::Int(_)
+            | TyKind::Uint(_)
+            | TyKind::Float(_)
+            | TyKind::Adt(_, _)
+            | TyKind::Foreign(_)
+            | TyKind::Str
+            | TyKind::Array(_, _)
+            | TyKind::Slice(_)
+            | TyKind::RawPtr(_)
+            | TyKind::Ref(_, _, _)
+            | TyKind::FnDef(_, _)
+            | TyKind::FnPtr(_)
+            | TyKind::Dynamic(_, _, _)
+            | TyKind::Closure(_, _)
+            | TyKind::Coroutine(_, _, _)
+            | TyKind::CoroutineWitness(..)
+            | TyKind::Never
+            | TyKind::Tuple(_)
+            | TyKind::Alias(_, _)
+            | TyKind::Bound(_, _)
+            | TyKind::Error(_) => return t.super_fold_with(self),
+        };
+
+        let t = t.clone().into();
+        let var =
+            BoundVar::from(self.variables.iter().position(|v| v == &t).unwrap_or_else(|| {
+                let var = self.variables.len();
+                self.variables.push(t);
+                self.primitive_var_infos.push(CanonicalVarInfo { kind });
+                var
+            }));
+
+        self.interner().mk_bound_ty(self.binder_index, var)
+    }
+
+    fn fold_const(&mut self, c: I::Const) -> I::Const
+    where
+        I::Const: TypeSuperFoldable<I>,
+    {
+        let kind = match c.kind() {
+            ConstKind::Infer(i) => {
+                // FIXME: we should fold the ty too eventually
+                match i {
+                    InferConst::Var(vid) => {
+                        assert_eq!(
+                            self.infcx.root_ct_var(vid),
+                            vid,
+                            "region vid should have been resolved fully before canonicalization"
+                        );
+                        assert_eq!(
+                            self.infcx.probe_ct_var(vid),
+                            None,
+                            "region vid should have been resolved fully before canonicalization"
+                        );
+                        CanonicalVarKind::Const(self.infcx.universe_of_ct(vid).unwrap(), c.ty())
+                    }
+                    InferConst::EffectVar(_) => CanonicalVarKind::Effect,
+                    InferConst::Fresh(_) => todo!(),
+                }
+            }
+            ConstKind::Placeholder(placeholder) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
+                    Placeholder::new(placeholder.universe(), self.variables.len().into()),
+                    c.ty(),
+                ),
+                CanonicalizeMode::Response { .. } => {
+                    CanonicalVarKind::PlaceholderConst(placeholder, c.ty())
+                }
+            },
+            ConstKind::Param(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
+                    Placeholder::new(UniverseIndex::ROOT, self.variables.len().into()),
+                    c.ty(),
+                ),
+                CanonicalizeMode::Response { .. } => panic!("param ty in response: {c:?}"),
+            },
+            ConstKind::Bound(_, _)
+            | ConstKind::Unevaluated(_)
+            | ConstKind::Value(_)
+            | ConstKind::Error(_)
+            | ConstKind::Expr(_) => return c.super_fold_with(self),
+        };
+
+        let ty = c.ty();
+        let c = c.clone().into();
+        let var =
+            BoundVar::from(self.variables.iter().position(|v| v == &c).unwrap_or_else(|| {
+                let var = self.variables.len();
+                self.variables.push(c);
+                self.primitive_var_infos.push(CanonicalVarInfo { kind });
+                var
+            }));
+
+        self.interner().mk_bound_const(self.binder_index, var, ty)
+    }
+}
diff --git a/compiler/rustc_next_trait_solver/src/lib.rs b/compiler/rustc_next_trait_solver/src/lib.rs
new file mode 100644
index 00000000000..e5fc8f755e0
--- /dev/null
+++ b/compiler/rustc_next_trait_solver/src/lib.rs
@@ -0,0 +1 @@
+pub mod canonicalizer;
diff --git a/compiler/rustc_trait_selection/Cargo.toml b/compiler/rustc_trait_selection/Cargo.toml
index 7d098180b93..29c0d8b5ff1 100644
--- a/compiler/rustc_trait_selection/Cargo.toml
+++ b/compiler/rustc_trait_selection/Cargo.toml
@@ -15,6 +15,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_next_trait_solver = { path = "../rustc_next_trait_solver" }
 rustc_parse_format = { path = "../rustc_parse_format" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_session = { path = "../rustc_session" }
diff --git a/compiler/rustc_trait_selection/src/solve/canonicalize.rs b/compiler/rustc_trait_selection/src/solve/canonicalize.rs
deleted file mode 100644
index 469d1f5a9cb..00000000000
--- a/compiler/rustc_trait_selection/src/solve/canonicalize.rs
+++ /dev/null
@@ -1,428 +0,0 @@
-use std::cmp::Ordering;
-
-use crate::infer::InferCtxt;
-use rustc_middle::infer::canonical::Canonical;
-use rustc_middle::infer::canonical::CanonicalTyVarKind;
-use rustc_middle::infer::canonical::CanonicalVarInfo;
-use rustc_middle::infer::canonical::CanonicalVarInfos;
-use rustc_middle::infer::canonical::CanonicalVarKind;
-use rustc_middle::ty::BoundRegionKind::BrAnon;
-use rustc_middle::ty::BoundTyKind;
-use rustc_middle::ty::TyCtxt;
-use rustc_middle::ty::TypeVisitableExt;
-use rustc_middle::ty::{self, Ty};
-use rustc_middle::ty::{TypeFoldable, TypeFolder, TypeSuperFoldable};
-
-/// Whether we're canonicalizing a query input or the query response.
-///
-/// When canonicalizing an input we're in the context of the caller
-/// while canonicalizing the response happens in the context of the
-/// query.
-#[derive(Debug, Clone, Copy)]
-pub enum CanonicalizeMode {
-    Input,
-    /// FIXME: We currently return region constraints referring to
-    /// placeholders and inference variables from a binder instantiated
-    /// inside of the query.
-    ///
-    /// In the long term we should eagerly deal with these constraints
-    /// inside of the query and only propagate constraints which are
-    /// actually nameable by the caller.
-    Response {
-        /// The highest universe nameable by the caller.
-        ///
-        /// All variables in a universe nameable by the caller get mapped
-        /// to the root universe in the response and then mapped back to
-        /// their correct universe when applying the query response in the
-        /// context of the caller.
-        ///
-        /// This doesn't work for universes created inside of the query so
-        /// we do remember their universe in the response.
-        max_input_universe: ty::UniverseIndex,
-    },
-}
-
-pub struct Canonicalizer<'a, 'tcx> {
-    infcx: &'a InferCtxt<'tcx>,
-    canonicalize_mode: CanonicalizeMode,
-
-    variables: &'a mut Vec<ty::GenericArg<'tcx>>,
-    primitive_var_infos: Vec<CanonicalVarInfo<'tcx>>,
-    binder_index: ty::DebruijnIndex,
-}
-
-impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
-    #[instrument(level = "debug", skip(infcx), ret)]
-    pub fn canonicalize<T: TypeFoldable<TyCtxt<'tcx>>>(
-        infcx: &'a InferCtxt<'tcx>,
-        canonicalize_mode: CanonicalizeMode,
-        variables: &'a mut Vec<ty::GenericArg<'tcx>>,
-        value: T,
-    ) -> Canonical<'tcx, T> {
-        let mut canonicalizer = Canonicalizer {
-            infcx,
-            canonicalize_mode,
-
-            variables,
-            primitive_var_infos: Vec::new(),
-            binder_index: ty::INNERMOST,
-        };
-
-        let value = value.fold_with(&mut canonicalizer);
-        assert!(!value.has_infer(), "unexpected infer in {value:?}");
-        assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
-
-        let (max_universe, variables) = canonicalizer.finalize();
-
-        Canonical { max_universe, variables, value }
-    }
-
-    fn finalize(self) -> (ty::UniverseIndex, CanonicalVarInfos<'tcx>) {
-        let mut var_infos = self.primitive_var_infos;
-        // See the rustc-dev-guide section about how we deal with universes
-        // during canonicalization in the new solver.
-        match self.canonicalize_mode {
-            // We try to deduplicate as many query calls as possible and hide
-            // all information which should not matter for the solver.
-            //
-            // For this we compress universes as much as possible.
-            CanonicalizeMode::Input => {}
-            // When canonicalizing a response we map a universes already entered
-            // by the caller to the root universe and only return useful universe
-            // information for placeholders and inference variables created inside
-            // of the query.
-            CanonicalizeMode::Response { max_input_universe } => {
-                for var in var_infos.iter_mut() {
-                    let uv = var.universe();
-                    let new_uv = ty::UniverseIndex::from(
-                        uv.index().saturating_sub(max_input_universe.index()),
-                    );
-                    *var = var.with_updated_universe(new_uv);
-                }
-                let max_universe = var_infos
-                    .iter()
-                    .map(|info| info.universe())
-                    .max()
-                    .unwrap_or(ty::UniverseIndex::ROOT);
-
-                let var_infos = self.infcx.tcx.mk_canonical_var_infos(&var_infos);
-                return (max_universe, var_infos);
-            }
-        }
-
-        // Given a `var_infos` with existentials `En` and universals `Un` in
-        // universes `n`, this algorithm compresses them in place so that:
-        //
-        // - the new universe indices are as small as possible
-        // - we only create a new universe if we would otherwise put a placeholder in
-        //   the same compressed universe as an existential which cannot name it
-        //
-        // Let's walk through an example:
-        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 0
-        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 1
-        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 1, next_orig_uv: 2
-        // - var_infos: [E0, U1, E5, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 5
-        // - var_infos: [E0, U1, E1, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 6
-        // - var_infos: [E0, U1, E1, U1, E1, E2, U2], curr_compressed_uv: 2, next_orig_uv: -
-        //
-        // This algorithm runs in `O(n²)` where `n` is the number of different universe
-        // indices in the input. This should be fine as `n` is expected to be small.
-        let mut curr_compressed_uv = ty::UniverseIndex::ROOT;
-        let mut existential_in_new_uv = false;
-        let mut next_orig_uv = Some(ty::UniverseIndex::ROOT);
-        while let Some(orig_uv) = next_orig_uv.take() {
-            let mut update_uv = |var: &mut CanonicalVarInfo<'tcx>, orig_uv, is_existential| {
-                let uv = var.universe();
-                match uv.cmp(&orig_uv) {
-                    Ordering::Less => (), // Already updated
-                    Ordering::Equal => {
-                        if is_existential {
-                            existential_in_new_uv = true;
-                        } else if existential_in_new_uv {
-                            //  `var` is a placeholder from a universe which is not nameable
-                            // by an existential which we already put into the compressed
-                            // universe `curr_compressed_uv`. We therefore have to create a
-                            // new universe for `var`.
-                            curr_compressed_uv = curr_compressed_uv.next_universe();
-                            existential_in_new_uv = false;
-                        }
-
-                        *var = var.with_updated_universe(curr_compressed_uv);
-                    }
-                    Ordering::Greater => {
-                        // We can ignore this variable in this iteration. We only look at
-                        // universes which actually occur in the input for performance.
-                        //
-                        // For this we set `next_orig_uv` to the next smallest, not yet compressed,
-                        // universe of the input.
-                        if next_orig_uv.map_or(true, |curr_next_uv| uv.cannot_name(curr_next_uv)) {
-                            next_orig_uv = Some(uv);
-                        }
-                    }
-                }
-            };
-
-            // For each universe which occurs in the input, we first iterate over all
-            // placeholders and then over all inference variables.
-            //
-            // Whenever we compress the universe of a placeholder, no existential with
-            // an already compressed universe can name that placeholder.
-            for is_existential in [false, true] {
-                for var in var_infos.iter_mut() {
-                    // We simply put all regions from the input into the highest
-                    // compressed universe, so we only deal with them at the end.
-                    if !var.is_region() {
-                        if is_existential == var.is_existential() {
-                            update_uv(var, orig_uv, is_existential)
-                        }
-                    }
-                }
-            }
-        }
-
-        for var in var_infos.iter_mut() {
-            if var.is_region() {
-                assert!(var.is_existential());
-                *var = var.with_updated_universe(curr_compressed_uv);
-            }
-        }
-
-        let var_infos = self.infcx.tcx.mk_canonical_var_infos(&var_infos);
-        (curr_compressed_uv, var_infos)
-    }
-}
-
-impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.infcx.tcx
-    }
-
-    fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
-    where
-        T: TypeFoldable<TyCtxt<'tcx>>,
-    {
-        self.binder_index.shift_in(1);
-        let t = t.super_fold_with(self);
-        self.binder_index.shift_out(1);
-        t
-    }
-
-    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        if let ty::ReVar(vid) = *r {
-            let resolved_region = self
-                .infcx
-                .inner
-                .borrow_mut()
-                .unwrap_region_constraints()
-                .opportunistic_resolve_var(self.infcx.tcx, vid);
-            assert_eq!(
-                r, resolved_region,
-                "region var should have been resolved, {r} -> {resolved_region}"
-            );
-        }
-
-        let kind = match *r {
-            ty::ReBound(..) => return r,
-
-            // We may encounter `ReStatic` in item signatures or the hidden type
-            // of an opaque. `ReErased` should only be encountered in the hidden
-            // type of an opaque for regions that are ignored for the purposes of
-            // captures.
-            //
-            // FIXME: We should investigate the perf implications of not uniquifying
-            // `ReErased`. We may be able to short-circuit registering region
-            // obligations if we encounter a `ReErased` on one side, for example.
-            ty::ReStatic | ty::ReErased => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
-                CanonicalizeMode::Response { .. } => return r,
-            },
-
-            ty::ReLateParam(_) | ty::ReEarlyParam(_) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
-                CanonicalizeMode::Response { .. } => bug!("unexpected region in response: {r:?}"),
-            },
-
-            ty::RePlaceholder(placeholder) => match self.canonicalize_mode {
-                // We canonicalize placeholder regions as existentials in query inputs.
-                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
-                CanonicalizeMode::Response { max_input_universe } => {
-                    // If we have a placeholder region inside of a query, it must be from
-                    // a new universe.
-                    if max_input_universe.can_name(placeholder.universe) {
-                        bug!("new placeholder in universe {max_input_universe:?}: {r:?}");
-                    }
-                    CanonicalVarKind::PlaceholderRegion(placeholder)
-                }
-            },
-
-            ty::ReVar(_) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
-                CanonicalizeMode::Response { .. } => {
-                    CanonicalVarKind::Region(self.infcx.universe_of_region(r))
-                }
-            },
-
-            ty::ReError(_) => return r,
-        };
-
-        let existing_bound_var = match self.canonicalize_mode {
-            CanonicalizeMode::Input => None,
-            CanonicalizeMode::Response { .. } => {
-                self.variables.iter().position(|&v| v == r.into()).map(ty::BoundVar::from)
-            }
-        };
-        let var = existing_bound_var.unwrap_or_else(|| {
-            let var = ty::BoundVar::from(self.variables.len());
-            self.variables.push(r.into());
-            self.primitive_var_infos.push(CanonicalVarInfo { kind });
-            var
-        });
-        let br = ty::BoundRegion { var, kind: BrAnon };
-        ty::Region::new_bound(self.interner(), self.binder_index, br)
-    }
-
-    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-        let kind = match *t.kind() {
-            ty::Infer(ty::TyVar(vid)) => {
-                assert_eq!(self.infcx.root_var(vid), vid, "ty vid should have been resolved");
-                let Err(ui) = self.infcx.probe_ty_var(vid) else {
-                    bug!("ty var should have been resolved: {t}");
-                };
-                CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui))
-            }
-            ty::Infer(ty::IntVar(vid)) => {
-                assert_eq!(self.infcx.opportunistic_resolve_int_var(vid), t);
-                CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
-            }
-            ty::Infer(ty::FloatVar(vid)) => {
-                assert_eq!(self.infcx.opportunistic_resolve_float_var(vid), t);
-                CanonicalVarKind::Ty(CanonicalTyVarKind::Float)
-            }
-            ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
-                bug!("fresh var during canonicalization: {t:?}")
-            }
-            ty::Placeholder(placeholder) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
-                    universe: placeholder.universe,
-                    bound: ty::BoundTy {
-                        var: ty::BoundVar::from_usize(self.variables.len()),
-                        kind: ty::BoundTyKind::Anon,
-                    },
-                }),
-                CanonicalizeMode::Response { .. } => CanonicalVarKind::PlaceholderTy(placeholder),
-            },
-            ty::Param(_) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
-                    universe: ty::UniverseIndex::ROOT,
-                    bound: ty::BoundTy {
-                        var: ty::BoundVar::from_usize(self.variables.len()),
-                        kind: ty::BoundTyKind::Anon,
-                    },
-                }),
-                CanonicalizeMode::Response { .. } => bug!("param ty in response: {t:?}"),
-            },
-            ty::Bool
-            | ty::Char
-            | ty::Int(_)
-            | ty::Uint(_)
-            | ty::Float(_)
-            | ty::Adt(_, _)
-            | ty::Foreign(_)
-            | ty::Str
-            | ty::Array(_, _)
-            | ty::Slice(_)
-            | ty::RawPtr(_)
-            | ty::Ref(_, _, _)
-            | ty::FnDef(_, _)
-            | ty::FnPtr(_)
-            | ty::Dynamic(_, _, _)
-            | ty::Closure(_, _)
-            | ty::Coroutine(_, _, _)
-            | ty::CoroutineWitness(..)
-            | ty::Never
-            | ty::Tuple(_)
-            | ty::Alias(_, _)
-            | ty::Bound(_, _)
-            | ty::Error(_) => return t.super_fold_with(self),
-        };
-
-        let var = ty::BoundVar::from(
-            self.variables.iter().position(|&v| v == t.into()).unwrap_or_else(|| {
-                let var = self.variables.len();
-                self.variables.push(t.into());
-                self.primitive_var_infos.push(CanonicalVarInfo { kind });
-                var
-            }),
-        );
-        let bt = ty::BoundTy { var, kind: BoundTyKind::Anon };
-        Ty::new_bound(self.infcx.tcx, self.binder_index, bt)
-    }
-
-    fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
-        let kind = match c.kind() {
-            ty::ConstKind::Infer(ty::InferConst::Var(vid)) => {
-                assert_eq!(
-                    self.infcx.root_const_var(vid),
-                    vid,
-                    "const var should have been resolved"
-                );
-                let Err(ui) = self.infcx.probe_const_var(vid) else {
-                    bug!("const var should have been resolved");
-                };
-                // FIXME: we should fold this ty eventually
-                CanonicalVarKind::Const(ui, c.ty())
-            }
-            ty::ConstKind::Infer(ty::InferConst::EffectVar(vid)) => {
-                assert_eq!(
-                    self.infcx.root_effect_var(vid),
-                    vid,
-                    "effect var should have been resolved"
-                );
-                let None = self.infcx.probe_effect_var(vid) else {
-                    bug!("effect var should have been resolved");
-                };
-                CanonicalVarKind::Effect
-            }
-            ty::ConstKind::Infer(ty::InferConst::Fresh(_)) => {
-                bug!("fresh var during canonicalization: {c:?}")
-            }
-            ty::ConstKind::Placeholder(placeholder) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
-                    ty::Placeholder {
-                        universe: placeholder.universe,
-                        bound: ty::BoundVar::from(self.variables.len()),
-                    },
-                    c.ty(),
-                ),
-                CanonicalizeMode::Response { .. } => {
-                    CanonicalVarKind::PlaceholderConst(placeholder, c.ty())
-                }
-            },
-            ty::ConstKind::Param(_) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
-                    ty::Placeholder {
-                        universe: ty::UniverseIndex::ROOT,
-                        bound: ty::BoundVar::from(self.variables.len()),
-                    },
-                    c.ty(),
-                ),
-                CanonicalizeMode::Response { .. } => bug!("param ty in response: {c:?}"),
-            },
-            ty::ConstKind::Bound(_, _)
-            | ty::ConstKind::Unevaluated(_)
-            | ty::ConstKind::Value(_)
-            | ty::ConstKind::Error(_)
-            | ty::ConstKind::Expr(_) => return c.super_fold_with(self),
-        };
-
-        let var = ty::BoundVar::from(
-            self.variables.iter().position(|&v| v == c.into()).unwrap_or_else(|| {
-                let var = self.variables.len();
-                self.variables.push(c.into());
-                self.primitive_var_infos.push(CanonicalVarInfo { kind });
-                var
-            }),
-        );
-        ty::Const::new_bound(self.infcx.tcx, self.binder_index, var, c.ty())
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
index da2f16e9760..7457ba837f5 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
@@ -9,7 +9,6 @@
 //!
 //! [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
 use super::{CanonicalInput, Certainty, EvalCtxt, Goal};
-use crate::solve::canonicalize::{CanonicalizeMode, Canonicalizer};
 use crate::solve::{
     inspect, response_no_constraints_raw, CanonicalResponse, QueryResult, Response,
 };
@@ -27,6 +26,7 @@ use rustc_middle::traits::solve::{
 };
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::{self, BoundVar, GenericArgKind, Ty, TyCtxt, TypeFoldable};
+use rustc_next_trait_solver::canonicalizer::{CanonicalizeMode, Canonicalizer};
 use rustc_span::DUMMY_SP;
 use std::iter;
 use std::ops::Deref;
diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs
index cf3f94e26e4..0ab099ef0c8 100644
--- a/compiler/rustc_trait_selection/src/solve/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/mod.rs
@@ -30,7 +30,6 @@ use rustc_middle::ty::{
 
 mod alias_relate;
 mod assembly;
-mod canonicalize;
 mod eval_ctxt;
 mod fulfill;
 pub mod inspect;
diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs
index d2768703297..514658046b7 100644
--- a/compiler/rustc_type_ir/src/canonical.rs
+++ b/compiler/rustc_type_ir/src/canonical.rs
@@ -162,7 +162,7 @@ impl<I: Interner> CanonicalVarInfo<I> {
     }
 
     #[must_use]
-    pub fn with_updated_universe(self, ui: UniverseIndex) -> CanonicalVarInfo<I> {
+    pub fn with_updated_universe(&self, ui: UniverseIndex) -> CanonicalVarInfo<I> {
         CanonicalVarInfo { kind: self.kind.with_updated_universe(ui) }
     }
 
@@ -324,13 +324,13 @@ impl<I: Interner> CanonicalVarKind<I> {
     ///
     /// In case this is a float or int variable, this causes an ICE if
     /// the updated universe is not the root.
-    pub fn with_updated_universe(self, ui: UniverseIndex) -> CanonicalVarKind<I> {
+    pub fn with_updated_universe(&self, ui: UniverseIndex) -> CanonicalVarKind<I> {
         match self {
             CanonicalVarKind::Ty(CanonicalTyVarKind::General(_)) => {
                 CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui))
             }
             CanonicalVarKind::Region(_) => CanonicalVarKind::Region(ui),
-            CanonicalVarKind::Const(_, ty) => CanonicalVarKind::Const(ui, ty),
+            CanonicalVarKind::Const(_, ty) => CanonicalVarKind::Const(ui, ty.clone()),
 
             CanonicalVarKind::PlaceholderTy(placeholder) => {
                 CanonicalVarKind::PlaceholderTy(placeholder.with_updated_universe(ui))
@@ -339,12 +339,15 @@ impl<I: Interner> CanonicalVarKind<I> {
                 CanonicalVarKind::PlaceholderRegion(placeholder.with_updated_universe(ui))
             }
             CanonicalVarKind::PlaceholderConst(placeholder, ty) => {
-                CanonicalVarKind::PlaceholderConst(placeholder.with_updated_universe(ui), ty)
+                CanonicalVarKind::PlaceholderConst(
+                    placeholder.with_updated_universe(ui),
+                    ty.clone(),
+                )
             }
             CanonicalVarKind::Ty(CanonicalTyVarKind::Int | CanonicalTyVarKind::Float)
             | CanonicalVarKind::Effect => {
                 assert_eq!(ui, UniverseIndex::ROOT);
-                self
+                self.clone()
             }
         }
     }
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index 409033a2d8d..fb5b2a4437f 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -146,15 +146,15 @@ impl<I: Interner> DebugWithInfcx<I> for InferConst {
         this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
-        match this.infcx.universe_of_ct(*this.data) {
-            None => write!(f, "{:?}", this.data),
-            Some(universe) => match *this.data {
-                InferConst::Var(vid) => write!(f, "?{}_{}c", vid.index(), universe.index()),
-                InferConst::EffectVar(vid) => write!(f, "?{}_{}e", vid.index(), universe.index()),
-                InferConst::Fresh(_) => {
-                    unreachable!()
-                }
+        match *this.data {
+            InferConst::Var(vid) => match this.infcx.universe_of_ct(vid) {
+                None => write!(f, "{:?}", this.data),
+                Some(universe) => write!(f, "?{}_{}c", vid.index(), universe.index()),
             },
+            InferConst::EffectVar(vid) => write!(f, "?{}e", vid.index()),
+            InferConst::Fresh(_) => {
+                unreachable!()
+            }
         }
     }
 }
diff --git a/compiler/rustc_type_ir/src/debug.rs b/compiler/rustc_type_ir/src/debug.rs
index db29ec9da8c..1732d311cb4 100644
--- a/compiler/rustc_type_ir/src/debug.rs
+++ b/compiler/rustc_type_ir/src/debug.rs
@@ -1,35 +1,56 @@
-use crate::{InferConst, InferTy, Interner, UniverseIndex};
+use crate::{ConstVid, InferCtxtLike, Interner, TyVid, UniverseIndex};
 
 use core::fmt;
 use std::marker::PhantomData;
 
-pub trait InferCtxtLike {
-    type Interner: Interner;
+pub struct NoInfcx<I>(PhantomData<I>);
 
-    fn universe_of_ty(&self, ty: InferTy) -> Option<UniverseIndex>;
+impl<I: Interner> InferCtxtLike for NoInfcx<I> {
+    type Interner = I;
 
-    fn universe_of_lt(
-        &self,
-        lt: <Self::Interner as Interner>::InferRegion,
-    ) -> Option<UniverseIndex>;
+    fn interner(&self) -> Self::Interner {
+        unreachable!()
+    }
 
-    fn universe_of_ct(&self, ct: InferConst) -> Option<UniverseIndex>;
-}
+    fn universe_of_ty(&self, _ty: TyVid) -> Option<UniverseIndex> {
+        None
+    }
 
-pub struct NoInfcx<I>(PhantomData<I>);
+    fn universe_of_lt(&self, _lt: <I as Interner>::InferRegion) -> Option<UniverseIndex> {
+        None
+    }
 
-impl<I: Interner> InferCtxtLike for NoInfcx<I> {
-    type Interner = I;
+    fn universe_of_ct(&self, _ct: ConstVid) -> Option<UniverseIndex> {
+        None
+    }
 
-    fn universe_of_ty(&self, _ty: InferTy) -> Option<UniverseIndex> {
+    fn root_ty_var(&self, vid: TyVid) -> TyVid {
+        vid
+    }
+
+    fn probe_ty_var(&self, _vid: TyVid) -> Option<<Self::Interner as Interner>::Ty> {
         None
     }
 
-    fn universe_of_ct(&self, _ct: InferConst) -> Option<UniverseIndex> {
+    fn root_lt_var(
+        &self,
+        vid: <Self::Interner as Interner>::InferRegion,
+    ) -> <Self::Interner as Interner>::InferRegion {
+        vid
+    }
+
+    fn probe_lt_var(
+        &self,
+        _vid: <Self::Interner as Interner>::InferRegion,
+    ) -> Option<<Self::Interner as Interner>::Region> {
         None
     }
 
-    fn universe_of_lt(&self, _lt: <I as Interner>::InferRegion) -> Option<UniverseIndex> {
+    fn root_ct_var(&self, vid: ConstVid) -> ConstVid {
+        vid
+    }
+
+    fn probe_ct_var(&self, _vid: ConstVid) -> Option<<Self::Interner as Interner>::Const> {
         None
     }
 }
diff --git a/compiler/rustc_type_ir/src/infcx.rs b/compiler/rustc_type_ir/src/infcx.rs
new file mode 100644
index 00000000000..681f129a50b
--- /dev/null
+++ b/compiler/rustc_type_ir/src/infcx.rs
@@ -0,0 +1,40 @@
+use crate::{ConstVid, Interner, TyVid, UniverseIndex};
+
+pub trait InferCtxtLike {
+    type Interner: Interner;
+
+    fn interner(&self) -> Self::Interner;
+
+    fn universe_of_ty(&self, ty: TyVid) -> Option<UniverseIndex>;
+
+    /// Resolve `TyVid` to its root `TyVid`.
+    fn root_ty_var(&self, vid: TyVid) -> TyVid;
+
+    /// Resolve `TyVid` to its inferred type, if it has been equated with a non-infer type.
+    fn probe_ty_var(&self, vid: TyVid) -> Option<<Self::Interner as Interner>::Ty>;
+
+    fn universe_of_lt(
+        &self,
+        lt: <Self::Interner as Interner>::InferRegion,
+    ) -> Option<UniverseIndex>;
+
+    /// Resolve `InferRegion` to its root `InferRegion`.
+    fn root_lt_var(
+        &self,
+        vid: <Self::Interner as Interner>::InferRegion,
+    ) -> <Self::Interner as Interner>::InferRegion;
+
+    /// Resolve `InferRegion` to its inferred region, if it has been equated with a non-infer region.
+    fn probe_lt_var(
+        &self,
+        vid: <Self::Interner as Interner>::InferRegion,
+    ) -> Option<<Self::Interner as Interner>::Region>;
+
+    fn universe_of_ct(&self, ct: ConstVid) -> Option<UniverseIndex>;
+
+    /// Resolve `ConstVid` to its root `ConstVid`.
+    fn root_ct_var(&self, vid: ConstVid) -> ConstVid;
+
+    /// Resolve `ConstVid` to its inferred type, if it has been equated with a non-infer type.
+    fn probe_ct_var(&self, vid: ConstVid) -> Option<<Self::Interner as Interner>::Const>;
+}
diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs
index 300b0bf090d..0acd576050f 100644
--- a/compiler/rustc_type_ir/src/interner.rs
+++ b/compiler/rustc_type_ir/src/interner.rs
@@ -2,8 +2,12 @@ use smallvec::SmallVec;
 use std::fmt::Debug;
 use std::hash::Hash;
 
-use crate::{BoundVar, DebugWithInfcx, Mutability, UniverseIndex};
+use crate::{
+    BoundVar, CanonicalVarInfo, ConstKind, DebruijnIndex, DebugWithInfcx, Mutability, RegionKind,
+    TyKind, UniverseIndex,
+};
 
+#[allow(rustc::usage_of_ty_tykind)]
 pub trait Interner: Sized {
     type DefId: Clone + Debug + Hash + Ord;
     type AdtDef: Clone + Debug + Hash + Ord;
@@ -18,10 +22,15 @@ pub trait Interner: Sized {
 
     type Binder<T>;
     type TypeAndMut: Clone + Debug + Hash + Ord;
-    type CanonicalVars: Clone + Debug + Hash + Eq;
+    type CanonicalVars: Clone + Debug + Hash + Eq + IntoIterator<Item = CanonicalVarInfo<Self>>;
 
     // Kinds of tys
-    type Ty: Clone + DebugWithInfcx<Self> + Hash + Ord;
+    type Ty: Clone
+        + DebugWithInfcx<Self>
+        + Hash
+        + Ord
+        + Into<Self::GenericArg>
+        + IntoKind<Kind = TyKind<Self>>;
     type Tys: Clone + Debug + Hash + Ord + IntoIterator<Item = Self::Ty>;
     type AliasTy: Clone + DebugWithInfcx<Self> + Hash + Ord;
     type ParamTy: Clone + Debug + Hash + Ord;
@@ -35,7 +44,13 @@ pub trait Interner: Sized {
     type AllocId: Clone + Debug + Hash + Ord;
 
     // Kinds of consts
-    type Const: Clone + DebugWithInfcx<Self> + Hash + Ord;
+    type Const: Clone
+        + DebugWithInfcx<Self>
+        + Hash
+        + Ord
+        + Into<Self::GenericArg>
+        + IntoKind<Kind = ConstKind<Self>>
+        + ConstTy<Self>;
     type AliasConst: Clone + DebugWithInfcx<Self> + Hash + Ord;
     type PlaceholderConst: Clone + Debug + Hash + Ord + Placeholder;
     type ParamConst: Clone + Debug + Hash + Ord;
@@ -44,10 +59,15 @@ pub trait Interner: Sized {
     type ExprConst: Clone + DebugWithInfcx<Self> + Hash + Ord;
 
     // Kinds of regions
-    type Region: Clone + DebugWithInfcx<Self> + Hash + Ord;
+    type Region: Clone
+        + DebugWithInfcx<Self>
+        + Hash
+        + Ord
+        + Into<Self::GenericArg>
+        + IntoKind<Kind = RegionKind<Self>>;
     type EarlyParamRegion: Clone + Debug + Hash + Ord;
-    type BoundRegion: Clone + Debug + Hash + Ord;
     type LateParamRegion: Clone + Debug + Hash + Ord;
+    type BoundRegion: Clone + Debug + Hash + Ord;
     type InferRegion: Clone + DebugWithInfcx<Self> + Hash + Ord;
     type PlaceholderRegion: Clone + Debug + Hash + Ord + Placeholder;
 
@@ -63,6 +83,12 @@ pub trait Interner: Sized {
     type ClosureKind: Clone + Debug + Hash + Eq;
 
     fn ty_and_mut_to_parts(ty_and_mut: Self::TypeAndMut) -> (Self::Ty, Mutability);
+
+    fn mk_canonical_var_infos(&self, infos: &[CanonicalVarInfo<Self>]) -> Self::CanonicalVars;
+
+    fn mk_bound_ty(&self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Ty;
+    fn mk_bound_region(&self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Region;
+    fn mk_bound_const(&self, debruijn: DebruijnIndex, var: BoundVar, ty: Self::Ty) -> Self::Const;
 }
 
 /// Common capabilities of placeholder kinds
@@ -70,7 +96,19 @@ pub trait Placeholder {
     fn universe(&self) -> UniverseIndex;
     fn var(&self) -> BoundVar;
 
-    fn with_updated_universe(self, ui: UniverseIndex) -> Self;
+    fn with_updated_universe(&self, ui: UniverseIndex) -> Self;
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self;
+}
+
+pub trait IntoKind {
+    type Kind;
+
+    fn kind(&self) -> Self::Kind;
+}
+
+pub trait ConstTy<I: Interner> {
+    fn ty(&self) -> I::Ty;
 }
 
 /// Imagine you have a function `F: FnOnce(&[T]) -> R`, plus an iterator `iter`
diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs
index 2aeb4230bb8..1bcb1fcff35 100644
--- a/compiler/rustc_type_ir/src/lib.rs
+++ b/compiler/rustc_type_ir/src/lib.rs
@@ -35,6 +35,7 @@ mod canonical;
 mod const_kind;
 mod debug;
 mod flags;
+mod infcx;
 mod interner;
 mod predicate_kind;
 mod region_kind;
@@ -43,8 +44,9 @@ pub use canonical::*;
 #[cfg(feature = "nightly")]
 pub use codec::*;
 pub use const_kind::*;
-pub use debug::{DebugWithInfcx, InferCtxtLike, WithInfcx};
+pub use debug::{DebugWithInfcx, WithInfcx};
 pub use flags::*;
+pub use infcx::InferCtxtLike;
 pub use interner::*;
 pub use predicate_kind::*;
 pub use region_kind::*;
@@ -337,6 +339,12 @@ impl UniverseIndex {
     }
 }
 
+impl Default for UniverseIndex {
+    fn default() -> Self {
+        Self::ROOT
+    }
+}
+
 rustc_index::newtype_index! {
     #[cfg_attr(feature = "nightly", derive(HashStable_NoContext))]
     #[encodable]
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index 3d4e7f77a4f..9d3a578c3e4 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -820,15 +820,15 @@ impl<I: Interner> DebugWithInfcx<I> for InferTy {
         this: WithInfcx<'_, Infcx, &Self>,
         f: &mut fmt::Formatter<'_>,
     ) -> fmt::Result {
-        use InferTy::*;
-        match this.infcx.universe_of_ty(*this.data) {
-            None => write!(f, "{:?}", this.data),
-            Some(universe) => match *this.data {
-                TyVar(ty_vid) => write!(f, "?{}_{}t", ty_vid.index(), universe.index()),
-                IntVar(_) | FloatVar(_) | FreshTy(_) | FreshIntTy(_) | FreshFloatTy(_) => {
-                    unreachable!()
+        match this.data {
+            InferTy::TyVar(vid) => {
+                if let Some(universe) = this.infcx.universe_of_ty(*vid) {
+                    write!(f, "?{}_{}t", vid.index(), universe.index())
+                } else {
+                    write!(f, "{:?}", this.data)
                 }
-            },
+            }
+            _ => write!(f, "{:?}", this.data),
         }
     }
 }
-- 
cgit 1.4.1-3-g733a5


From 1f5895b3e31e10fb9121c751b5aedc31dfe3d92f Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Tue, 5 Dec 2023 18:10:23 +0000
Subject: Feedback

- Take more things by self, not &self
- Clone more things
- Rework namespacing so we can use `ty::` in the canonicalizer
---
 compiler/rustc_middle/src/ty/consts.rs             |   8 +-
 compiler/rustc_middle/src/ty/context.rs            |  31 ++--
 compiler/rustc_middle/src/ty/mod.rs                |  39 ++---
 compiler/rustc_middle/src/ty/sty.rs                |   4 +-
 .../rustc_next_trait_solver/src/canonicalizer.rs   | 195 ++++++++++-----------
 compiler/rustc_type_ir/src/canonical.rs            |  25 ++-
 compiler/rustc_type_ir/src/const_kind.rs           |   2 +-
 compiler/rustc_type_ir/src/debug.rs                |  16 +-
 compiler/rustc_type_ir/src/interner.rs             | 100 +++++------
 compiler/rustc_type_ir/src/lib.rs                  |   6 +
 compiler/rustc_type_ir/src/ty_kind.rs              |   8 +-
 11 files changed, 205 insertions(+), 229 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index 78cc2f91796..293df4f691d 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -29,14 +29,14 @@ pub struct Const<'tcx>(pub(super) Interned<'tcx, WithCachedTypeInfo<ConstData<'t
 impl<'tcx> IntoKind for Const<'tcx> {
     type Kind = ConstKind<'tcx>;
 
-    fn kind(&self) -> ConstKind<'tcx> {
-        (*self).kind().clone()
+    fn kind(self) -> ConstKind<'tcx> {
+        self.kind().clone()
     }
 }
 
 impl<'tcx> ConstTy<TyCtxt<'tcx>> for Const<'tcx> {
-    fn ty(&self) -> Ty<'tcx> {
-        (*self).ty()
+    fn ty(self) -> Ty<'tcx> {
+        self.ty()
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index a39acc9859f..050f074ee0a 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -132,40 +132,29 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
         (ty, mutbl)
     }
 
-    fn mk_canonical_var_infos(
-        &self,
-        infos: &[rustc_type_ir::CanonicalVarInfo<Self>],
-    ) -> Self::CanonicalVars {
-        (*self).mk_canonical_var_infos(infos)
+    fn mk_canonical_var_infos(self, infos: &[ty::CanonicalVarInfo<Self>]) -> Self::CanonicalVars {
+        self.mk_canonical_var_infos(infos)
     }
 
-    fn mk_bound_ty(
-        &self,
-        debruijn: rustc_type_ir::DebruijnIndex,
-        var: rustc_type_ir::BoundVar,
-    ) -> Self::Ty {
-        Ty::new_bound(*self, debruijn, ty::BoundTy { var, kind: ty::BoundTyKind::Anon })
+    fn mk_bound_ty(self, debruijn: ty::DebruijnIndex, var: ty::BoundVar) -> Self::Ty {
+        Ty::new_bound(self, debruijn, ty::BoundTy { var, kind: ty::BoundTyKind::Anon })
     }
 
-    fn mk_bound_region(
-        &self,
-        debruijn: rustc_type_ir::DebruijnIndex,
-        var: rustc_type_ir::BoundVar,
-    ) -> Self::Region {
+    fn mk_bound_region(self, debruijn: ty::DebruijnIndex, var: ty::BoundVar) -> Self::Region {
         Region::new_bound(
-            *self,
+            self,
             debruijn,
             ty::BoundRegion { var, kind: ty::BoundRegionKind::BrAnon },
         )
     }
 
     fn mk_bound_const(
-        &self,
-        debruijn: rustc_type_ir::DebruijnIndex,
-        var: rustc_type_ir::BoundVar,
+        self,
+        debruijn: ty::DebruijnIndex,
+        var: ty::BoundVar,
         ty: Self::Ty,
     ) -> Self::Const {
-        Const::new_bound(*self, debruijn, var, ty)
+        Const::new_bound(self, debruijn, var, ty)
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 6aaa363eb01..71ff7021ca5 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -65,15 +65,10 @@ use std::ops::ControlFlow;
 use std::{fmt, str};
 
 pub use crate::ty::diagnostics::*;
-pub use rustc_type_ir::AliasKind::*;
 pub use rustc_type_ir::ConstKind::{
     Bound as BoundCt, Error as ErrorCt, Expr as ExprCt, Infer as InferCt, Param as ParamCt,
     Placeholder as PlaceholderCt, Unevaluated, Value,
 };
-pub use rustc_type_ir::DynKind::*;
-pub use rustc_type_ir::InferTy::*;
-pub use rustc_type_ir::RegionKind::*;
-pub use rustc_type_ir::TyKind::*;
 pub use rustc_type_ir::*;
 
 pub use self::binding::BindingMode;
@@ -477,8 +472,8 @@ pub struct Ty<'tcx>(Interned<'tcx, WithCachedTypeInfo<TyKind<'tcx>>>);
 impl<'tcx> IntoKind for Ty<'tcx> {
     type Kind = TyKind<'tcx>;
 
-    fn kind(&self) -> TyKind<'tcx> {
-        (*self).kind().clone()
+    fn kind(self) -> TyKind<'tcx> {
+        self.kind().clone()
     }
 }
 
@@ -1553,17 +1548,17 @@ pub struct Placeholder<T> {
 
 pub type PlaceholderRegion = Placeholder<BoundRegion>;
 
-impl rustc_type_ir::Placeholder for PlaceholderRegion {
-    fn universe(&self) -> UniverseIndex {
+impl PlaceholderLike for PlaceholderRegion {
+    fn universe(self) -> UniverseIndex {
         self.universe
     }
 
-    fn var(&self) -> BoundVar {
+    fn var(self) -> BoundVar {
         self.bound.var
     }
 
-    fn with_updated_universe(&self, ui: UniverseIndex) -> Self {
-        Placeholder { universe: ui, ..*self }
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..self }
     }
 
     fn new(ui: UniverseIndex, var: BoundVar) -> Self {
@@ -1573,17 +1568,17 @@ impl rustc_type_ir::Placeholder for PlaceholderRegion {
 
 pub type PlaceholderType = Placeholder<BoundTy>;
 
-impl rustc_type_ir::Placeholder for PlaceholderType {
-    fn universe(&self) -> UniverseIndex {
+impl PlaceholderLike for PlaceholderType {
+    fn universe(self) -> UniverseIndex {
         self.universe
     }
 
-    fn var(&self) -> BoundVar {
+    fn var(self) -> BoundVar {
         self.bound.var
     }
 
-    fn with_updated_universe(&self, ui: UniverseIndex) -> Self {
-        Placeholder { universe: ui, ..*self }
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..self }
     }
 
     fn new(ui: UniverseIndex, var: BoundVar) -> Self {
@@ -1600,17 +1595,17 @@ pub struct BoundConst<'tcx> {
 
 pub type PlaceholderConst = Placeholder<BoundVar>;
 
-impl rustc_type_ir::Placeholder for PlaceholderConst {
-    fn universe(&self) -> UniverseIndex {
+impl PlaceholderLike for PlaceholderConst {
+    fn universe(self) -> UniverseIndex {
         self.universe
     }
 
-    fn var(&self) -> BoundVar {
+    fn var(self) -> BoundVar {
         self.bound
     }
 
-    fn with_updated_universe(&self, ui: UniverseIndex) -> Self {
-        Placeholder { universe: ui, ..*self }
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..self }
     }
 
     fn new(ui: UniverseIndex, var: BoundVar) -> Self {
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index ed2f8f0a6ab..50a1b85b169 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -1480,8 +1480,8 @@ pub struct Region<'tcx>(pub Interned<'tcx, RegionKind<'tcx>>);
 impl<'tcx> IntoKind for Region<'tcx> {
     type Kind = RegionKind<'tcx>;
 
-    fn kind(&self) -> RegionKind<'tcx> {
-        **self
+    fn kind(self) -> RegionKind<'tcx> {
+        *self
     }
 }
 
diff --git a/compiler/rustc_next_trait_solver/src/canonicalizer.rs b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
index c2e6e82f2af..cb1f328577d 100644
--- a/compiler/rustc_next_trait_solver/src/canonicalizer.rs
+++ b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
@@ -2,9 +2,8 @@ use std::cmp::Ordering;
 
 use rustc_type_ir::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_type_ir::{
-    BoundVar, Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, ConstKind,
-    ConstTy, DebruijnIndex, InferConst, InferCtxtLike, InferTy, Interner, IntoKind, Placeholder,
-    RegionKind, TyKind, UniverseIndex, INNERMOST,
+    self as ty, Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, ConstTy,
+    InferCtxtLike, Interner, IntoKind, PlaceholderLike,
 };
 
 /// Whether we're canonicalizing a query input or the query response.
@@ -32,33 +31,33 @@ pub enum CanonicalizeMode {
         ///
         /// This doesn't work for universes created inside of the query so
         /// we do remember their universe in the response.
-        max_input_universe: UniverseIndex,
+        max_input_universe: ty::UniverseIndex,
     },
 }
 
-pub struct Canonicalizer<'a, Infcx: InferCtxtLike> {
+pub struct Canonicalizer<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> {
     infcx: &'a Infcx,
     canonicalize_mode: CanonicalizeMode,
 
-    variables: &'a mut Vec<<Infcx::Interner as Interner>::GenericArg>,
-    primitive_var_infos: Vec<CanonicalVarInfo<Infcx::Interner>>,
-    binder_index: DebruijnIndex,
+    variables: &'a mut Vec<I::GenericArg>,
+    primitive_var_infos: Vec<CanonicalVarInfo<I>>,
+    binder_index: ty::DebruijnIndex,
 }
 
-impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infcx> {
+impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infcx, I> {
     pub fn canonicalize<T: TypeFoldable<I>>(
         infcx: &'a Infcx,
         canonicalize_mode: CanonicalizeMode,
-        variables: &'a mut Vec<<I as Interner>::GenericArg>,
+        variables: &'a mut Vec<I::GenericArg>,
         value: T,
-    ) -> Canonical<I, T> {
+    ) -> ty::Canonical<I, T> {
         let mut canonicalizer = Canonicalizer {
             infcx,
             canonicalize_mode,
 
             variables,
             primitive_var_infos: Vec::new(),
-            binder_index: INNERMOST,
+            binder_index: ty::INNERMOST,
         };
 
         let value = value.fold_with(&mut canonicalizer);
@@ -71,7 +70,7 @@ impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infc
         Canonical { max_universe, variables, value }
     }
 
-    fn finalize(self) -> (UniverseIndex, <I as Interner>::CanonicalVars) {
+    fn finalize(self) -> (ty::UniverseIndex, I::CanonicalVars) {
         let mut var_infos = self.primitive_var_infos;
         // See the rustc-dev-guide section about how we deal with universes
         // during canonicalization in the new solver.
@@ -88,15 +87,16 @@ impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infc
             CanonicalizeMode::Response { max_input_universe } => {
                 for var in var_infos.iter_mut() {
                     let uv = var.universe();
-                    let new_uv =
-                        UniverseIndex::from(uv.index().saturating_sub(max_input_universe.index()));
+                    let new_uv = ty::UniverseIndex::from(
+                        uv.index().saturating_sub(max_input_universe.index()),
+                    );
                     *var = var.with_updated_universe(new_uv);
                 }
                 let max_universe = var_infos
                     .iter()
                     .map(|info| info.universe())
                     .max()
-                    .unwrap_or(UniverseIndex::ROOT);
+                    .unwrap_or(ty::UniverseIndex::ROOT);
 
                 let var_infos = self.infcx.interner().mk_canonical_var_infos(&var_infos);
                 return (max_universe, var_infos);
@@ -120,9 +120,9 @@ impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infc
         //
         // This algorithm runs in `O(n²)` where `n` is the number of different universe
         // indices in the input. This should be fine as `n` is expected to be small.
-        let mut curr_compressed_uv = UniverseIndex::ROOT;
+        let mut curr_compressed_uv = ty::UniverseIndex::ROOT;
         let mut existential_in_new_uv = false;
-        let mut next_orig_uv = Some(UniverseIndex::ROOT);
+        let mut next_orig_uv = Some(ty::UniverseIndex::ROOT);
         while let Some(orig_uv) = next_orig_uv.take() {
             let mut update_uv = |var: &mut CanonicalVarInfo<I>, orig_uv, is_existential| {
                 let uv = var.universe();
@@ -176,8 +176,7 @@ impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infc
         for var in var_infos.iter_mut() {
             if var.is_region() {
                 assert!(var.is_existential());
-                let compressed_var = var.with_updated_universe(curr_compressed_uv);
-                *var = compressed_var;
+                *var = var.with_updated_universe(curr_compressed_uv);
             }
         }
 
@@ -186,7 +185,9 @@ impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infc
     }
 }
 
-impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonicalizer<'_, Infcx> {
+impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I>
+    for Canonicalizer<'_, Infcx, I>
+{
     fn interner(&self) -> I {
         self.infcx.interner()
     }
@@ -204,7 +205,7 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonica
 
     fn fold_region(&mut self, r: I::Region) -> I::Region {
         let kind = match r.kind() {
-            RegionKind::ReBound(..) => return r,
+            ty::ReBound(..) => return r,
 
             // We may encounter `ReStatic` in item signatures or the hidden type
             // of an opaque. `ReErased` should only be encountered in the hidden
@@ -214,23 +215,21 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonica
             // FIXME: We should investigate the perf implications of not uniquifying
             // `ReErased`. We may be able to short-circuit registering region
             // obligations if we encounter a `ReErased` on one side, for example.
-            RegionKind::ReStatic | RegionKind::ReErased => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
+            ty::ReStatic | ty::ReErased => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
                 CanonicalizeMode::Response { .. } => return r,
             },
 
-            RegionKind::ReEarlyParam(_) | RegionKind::ReLateParam(_) => {
-                match self.canonicalize_mode {
-                    CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
-                    CanonicalizeMode::Response { .. } => {
-                        panic!("unexpected region in response: {r:?}")
-                    }
+            ty::ReEarlyParam(_) | ty::ReLateParam(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
+                CanonicalizeMode::Response { .. } => {
+                    panic!("unexpected region in response: {r:?}")
                 }
-            }
+            },
 
-            RegionKind::RePlaceholder(placeholder) => match self.canonicalize_mode {
+            ty::RePlaceholder(placeholder) => match self.canonicalize_mode {
                 // We canonicalize placeholder regions as existentials in query inputs.
-                CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
+                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
                 CanonicalizeMode::Response { max_input_universe } => {
                     // If we have a placeholder region inside of a query, it must be from
                     // a new universe.
@@ -241,38 +240,37 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonica
                 }
             },
 
-            RegionKind::ReVar(vid) => {
+            ty::ReVar(vid) => {
                 assert_eq!(
-                    self.infcx.root_lt_var(vid.clone()),
+                    self.infcx.root_lt_var(vid),
                     vid,
                     "region vid should have been resolved fully before canonicalization"
                 );
                 assert_eq!(
-                    self.infcx.probe_lt_var(vid.clone()),
+                    self.infcx.probe_lt_var(vid),
                     None,
                     "region vid should have been resolved fully before canonicalization"
                 );
 
                 match self.canonicalize_mode {
-                    CanonicalizeMode::Input => CanonicalVarKind::Region(UniverseIndex::ROOT),
+                    CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
                     CanonicalizeMode::Response { .. } => {
                         CanonicalVarKind::Region(self.infcx.universe_of_lt(vid).unwrap())
                     }
                 }
             }
-            RegionKind::ReError(_) => return r,
+            ty::ReError(_) => return r,
         };
 
         let existing_bound_var = match self.canonicalize_mode {
             CanonicalizeMode::Input => None,
             CanonicalizeMode::Response { .. } => {
-                let r = r.clone().into();
-                self.variables.iter().position(|v| v == &r).map(BoundVar::from)
+                self.variables.iter().position(|&v| v == r.into()).map(ty::BoundVar::from)
             }
         };
 
         let var = existing_bound_var.unwrap_or_else(|| {
-            let var = BoundVar::from(self.variables.len());
+            let var = ty::BoundVar::from(self.variables.len());
             self.variables.push(r.into());
             self.primitive_var_infos.push(CanonicalVarInfo { kind });
             var
@@ -286,8 +284,8 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonica
         I::Ty: TypeSuperFoldable<I>,
     {
         let kind = match t.kind() {
-            TyKind::Infer(i) => match i {
-                InferTy::TyVar(vid) => {
+            ty::Infer(i) => match i {
+                ty::TyVar(vid) => {
                     assert_eq!(
                         self.infcx.root_ty_var(vid),
                         vid,
@@ -305,59 +303,59 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonica
                             .unwrap_or_else(|| panic!("ty var should have been resolved: {t:?}")),
                     ))
                 }
-                InferTy::IntVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Int),
-                InferTy::FloatVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Float),
-                InferTy::FreshTy(_) | InferTy::FreshIntTy(_) | InferTy::FreshFloatTy(_) => {
+                ty::IntVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Int),
+                ty::FloatVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Float),
+                ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => {
                     todo!()
                 }
             },
-            TyKind::Placeholder(placeholder) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(Placeholder::new(
+            ty::Placeholder(placeholder) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(PlaceholderLike::new(
                     placeholder.universe(),
                     self.variables.len().into(),
                 )),
                 CanonicalizeMode::Response { .. } => CanonicalVarKind::PlaceholderTy(placeholder),
             },
-            TyKind::Param(_) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(Placeholder::new(
-                    UniverseIndex::ROOT,
+            ty::Param(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(PlaceholderLike::new(
+                    ty::UniverseIndex::ROOT,
                     self.variables.len().into(),
                 )),
                 CanonicalizeMode::Response { .. } => panic!("param ty in response: {t:?}"),
             },
-            TyKind::Bool
-            | TyKind::Char
-            | TyKind::Int(_)
-            | TyKind::Uint(_)
-            | TyKind::Float(_)
-            | TyKind::Adt(_, _)
-            | TyKind::Foreign(_)
-            | TyKind::Str
-            | TyKind::Array(_, _)
-            | TyKind::Slice(_)
-            | TyKind::RawPtr(_)
-            | TyKind::Ref(_, _, _)
-            | TyKind::FnDef(_, _)
-            | TyKind::FnPtr(_)
-            | TyKind::Dynamic(_, _, _)
-            | TyKind::Closure(_, _)
-            | TyKind::Coroutine(_, _, _)
-            | TyKind::CoroutineWitness(..)
-            | TyKind::Never
-            | TyKind::Tuple(_)
-            | TyKind::Alias(_, _)
-            | TyKind::Bound(_, _)
-            | TyKind::Error(_) => return t.super_fold_with(self),
+            ty::Bool
+            | ty::Char
+            | ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Adt(_, _)
+            | ty::Foreign(_)
+            | ty::Str
+            | ty::Array(_, _)
+            | ty::Slice(_)
+            | ty::RawPtr(_)
+            | ty::Ref(_, _, _)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Closure(_, _)
+            | ty::Coroutine(_, _, _)
+            | ty::CoroutineWitness(..)
+            | ty::Never
+            | ty::Tuple(_)
+            | ty::Alias(_, _)
+            | ty::Bound(_, _)
+            | ty::Error(_) => return t.super_fold_with(self),
         };
 
-        let t = t.clone().into();
-        let var =
-            BoundVar::from(self.variables.iter().position(|v| v == &t).unwrap_or_else(|| {
+        let var = ty::BoundVar::from(
+            self.variables.iter().position(|&v| v == t.into()).unwrap_or_else(|| {
                 let var = self.variables.len();
-                self.variables.push(t);
+                self.variables.push(t.into());
                 self.primitive_var_infos.push(CanonicalVarInfo { kind });
                 var
-            }));
+            }),
+        );
 
         self.interner().mk_bound_ty(self.binder_index, var)
     }
@@ -367,10 +365,10 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonica
         I::Const: TypeSuperFoldable<I>,
     {
         let kind = match c.kind() {
-            ConstKind::Infer(i) => {
+            ty::ConstKind::Infer(i) => {
                 // FIXME: we should fold the ty too eventually
                 match i {
-                    InferConst::Var(vid) => {
+                    ty::InferConst::Var(vid) => {
                         assert_eq!(
                             self.infcx.root_ct_var(vid),
                             vid,
@@ -383,43 +381,42 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> for Canonica
                         );
                         CanonicalVarKind::Const(self.infcx.universe_of_ct(vid).unwrap(), c.ty())
                     }
-                    InferConst::EffectVar(_) => CanonicalVarKind::Effect,
-                    InferConst::Fresh(_) => todo!(),
+                    ty::InferConst::EffectVar(_) => CanonicalVarKind::Effect,
+                    ty::InferConst::Fresh(_) => todo!(),
                 }
             }
-            ConstKind::Placeholder(placeholder) => match self.canonicalize_mode {
+            ty::ConstKind::Placeholder(placeholder) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
-                    Placeholder::new(placeholder.universe(), self.variables.len().into()),
+                    PlaceholderLike::new(placeholder.universe(), self.variables.len().into()),
                     c.ty(),
                 ),
                 CanonicalizeMode::Response { .. } => {
                     CanonicalVarKind::PlaceholderConst(placeholder, c.ty())
                 }
             },
-            ConstKind::Param(_) => match self.canonicalize_mode {
+            ty::ConstKind::Param(_) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
-                    Placeholder::new(UniverseIndex::ROOT, self.variables.len().into()),
+                    PlaceholderLike::new(ty::UniverseIndex::ROOT, self.variables.len().into()),
                     c.ty(),
                 ),
                 CanonicalizeMode::Response { .. } => panic!("param ty in response: {c:?}"),
             },
-            ConstKind::Bound(_, _)
-            | ConstKind::Unevaluated(_)
-            | ConstKind::Value(_)
-            | ConstKind::Error(_)
-            | ConstKind::Expr(_) => return c.super_fold_with(self),
+            ty::ConstKind::Bound(_, _)
+            | ty::ConstKind::Unevaluated(_)
+            | ty::ConstKind::Value(_)
+            | ty::ConstKind::Error(_)
+            | ty::ConstKind::Expr(_) => return c.super_fold_with(self),
         };
 
-        let ty = c.ty();
-        let c = c.clone().into();
-        let var =
-            BoundVar::from(self.variables.iter().position(|v| v == &c).unwrap_or_else(|| {
+        let var = ty::BoundVar::from(
+            self.variables.iter().position(|&v| v == c.into()).unwrap_or_else(|| {
                 let var = self.variables.len();
-                self.variables.push(c);
+                self.variables.push(c.into());
                 self.primitive_var_infos.push(CanonicalVarInfo { kind });
                 var
-            }));
+            }),
+        );
 
-        self.interner().mk_bound_const(self.binder_index, var, ty)
+        self.interner().mk_bound_const(self.binder_index, var, c.ty())
     }
 }
diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs
index 514658046b7..572c6f201d3 100644
--- a/compiler/rustc_type_ir/src/canonical.rs
+++ b/compiler/rustc_type_ir/src/canonical.rs
@@ -4,7 +4,7 @@ use std::ops::ControlFlow;
 
 use crate::fold::{FallibleTypeFolder, TypeFoldable};
 use crate::visit::{TypeVisitable, TypeVisitor};
-use crate::{Interner, Placeholder, UniverseIndex};
+use crate::{Interner, PlaceholderLike, UniverseIndex};
 
 /// A "canonicalized" type `V` is one where all free inference
 /// variables have been rewritten to "canonical vars". These are
@@ -157,12 +157,12 @@ where
 }
 
 impl<I: Interner> CanonicalVarInfo<I> {
-    pub fn universe(&self) -> UniverseIndex {
+    pub fn universe(self) -> UniverseIndex {
         self.kind.universe()
     }
 
     #[must_use]
-    pub fn with_updated_universe(&self, ui: UniverseIndex) -> CanonicalVarInfo<I> {
+    pub fn with_updated_universe(self, ui: UniverseIndex) -> CanonicalVarInfo<I> {
         CanonicalVarInfo { kind: self.kind.with_updated_universe(ui) }
     }
 
@@ -305,11 +305,11 @@ where
 }
 
 impl<I: Interner> CanonicalVarKind<I> {
-    pub fn universe(&self) -> UniverseIndex {
+    pub fn universe(self) -> UniverseIndex {
         match self {
-            CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)) => *ui,
-            CanonicalVarKind::Region(ui) => *ui,
-            CanonicalVarKind::Const(ui, _) => *ui,
+            CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)) => ui,
+            CanonicalVarKind::Region(ui) => ui,
+            CanonicalVarKind::Const(ui, _) => ui,
             CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.universe(),
             CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.universe(),
             CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.universe(),
@@ -324,13 +324,13 @@ impl<I: Interner> CanonicalVarKind<I> {
     ///
     /// In case this is a float or int variable, this causes an ICE if
     /// the updated universe is not the root.
-    pub fn with_updated_universe(&self, ui: UniverseIndex) -> CanonicalVarKind<I> {
+    pub fn with_updated_universe(self, ui: UniverseIndex) -> CanonicalVarKind<I> {
         match self {
             CanonicalVarKind::Ty(CanonicalTyVarKind::General(_)) => {
                 CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui))
             }
             CanonicalVarKind::Region(_) => CanonicalVarKind::Region(ui),
-            CanonicalVarKind::Const(_, ty) => CanonicalVarKind::Const(ui, ty.clone()),
+            CanonicalVarKind::Const(_, ty) => CanonicalVarKind::Const(ui, ty),
 
             CanonicalVarKind::PlaceholderTy(placeholder) => {
                 CanonicalVarKind::PlaceholderTy(placeholder.with_updated_universe(ui))
@@ -339,15 +339,12 @@ impl<I: Interner> CanonicalVarKind<I> {
                 CanonicalVarKind::PlaceholderRegion(placeholder.with_updated_universe(ui))
             }
             CanonicalVarKind::PlaceholderConst(placeholder, ty) => {
-                CanonicalVarKind::PlaceholderConst(
-                    placeholder.with_updated_universe(ui),
-                    ty.clone(),
-                )
+                CanonicalVarKind::PlaceholderConst(placeholder.with_updated_universe(ui), ty)
             }
             CanonicalVarKind::Ty(CanonicalTyVarKind::Int | CanonicalTyVarKind::Float)
             | CanonicalVarKind::Effect => {
                 assert_eq!(ui, UniverseIndex::ROOT);
-                self.clone()
+                self
             }
         }
     }
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index fb5b2a4437f..879de58f100 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -81,7 +81,7 @@ impl<I: Interner> DebugWithInfcx<I> for ConstKind<I> {
         match this.data {
             Param(param) => write!(f, "{param:?}"),
             Infer(var) => write!(f, "{:?}", &this.wrap(var)),
-            Bound(debruijn, var) => crate::debug_bound_var(f, *debruijn, var.clone()),
+            Bound(debruijn, var) => crate::debug_bound_var(f, *debruijn, var),
             Placeholder(placeholder) => write!(f, "{placeholder:?}"),
             Unevaluated(uv) => {
                 write!(f, "{:?}", &this.wrap(uv))
diff --git a/compiler/rustc_type_ir/src/debug.rs b/compiler/rustc_type_ir/src/debug.rs
index 1732d311cb4..8998001ec20 100644
--- a/compiler/rustc_type_ir/src/debug.rs
+++ b/compiler/rustc_type_ir/src/debug.rs
@@ -16,7 +16,7 @@ impl<I: Interner> InferCtxtLike for NoInfcx<I> {
         None
     }
 
-    fn universe_of_lt(&self, _lt: <I as Interner>::InferRegion) -> Option<UniverseIndex> {
+    fn universe_of_lt(&self, _lt: I::InferRegion) -> Option<UniverseIndex> {
         None
     }
 
@@ -28,21 +28,15 @@ impl<I: Interner> InferCtxtLike for NoInfcx<I> {
         vid
     }
 
-    fn probe_ty_var(&self, _vid: TyVid) -> Option<<Self::Interner as Interner>::Ty> {
+    fn probe_ty_var(&self, _vid: TyVid) -> Option<I::Ty> {
         None
     }
 
-    fn root_lt_var(
-        &self,
-        vid: <Self::Interner as Interner>::InferRegion,
-    ) -> <Self::Interner as Interner>::InferRegion {
+    fn root_lt_var(&self, vid: I::InferRegion) -> I::InferRegion {
         vid
     }
 
-    fn probe_lt_var(
-        &self,
-        _vid: <Self::Interner as Interner>::InferRegion,
-    ) -> Option<<Self::Interner as Interner>::Region> {
+    fn probe_lt_var(&self, _vid: I::InferRegion) -> Option<I::Region> {
         None
     }
 
@@ -50,7 +44,7 @@ impl<I: Interner> InferCtxtLike for NoInfcx<I> {
         vid
     }
 
-    fn probe_ct_var(&self, _vid: ConstVid) -> Option<<Self::Interner as Interner>::Const> {
+    fn probe_ct_var(&self, _vid: ConstVid) -> Option<I::Const> {
         None
     }
 }
diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs
index 0acd576050f..c262302133b 100644
--- a/compiler/rustc_type_ir/src/interner.rs
+++ b/compiler/rustc_type_ir/src/interner.rs
@@ -7,96 +7,96 @@ use crate::{
     TyKind, UniverseIndex,
 };
 
-#[allow(rustc::usage_of_ty_tykind)]
 pub trait Interner: Sized {
-    type DefId: Clone + Debug + Hash + Ord;
-    type AdtDef: Clone + Debug + Hash + Ord;
+    type DefId: Copy + Debug + Hash + Ord;
+    type AdtDef: Copy + Debug + Hash + Ord;
 
-    type GenericArgs: Clone
+    type GenericArgs: Copy
         + DebugWithInfcx<Self>
         + Hash
         + Ord
         + IntoIterator<Item = Self::GenericArg>;
-    type GenericArg: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type Term: Clone + Debug + Hash + Ord;
+    type GenericArg: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type Term: Copy + Debug + Hash + Ord;
 
     type Binder<T>;
-    type TypeAndMut: Clone + Debug + Hash + Ord;
-    type CanonicalVars: Clone + Debug + Hash + Eq + IntoIterator<Item = CanonicalVarInfo<Self>>;
+    type TypeAndMut: Copy + Debug + Hash + Ord;
+    type CanonicalVars: Copy + Debug + Hash + Eq + IntoIterator<Item = CanonicalVarInfo<Self>>;
 
     // Kinds of tys
-    type Ty: Clone
+    type Ty: Copy
         + DebugWithInfcx<Self>
         + Hash
         + Ord
         + Into<Self::GenericArg>
         + IntoKind<Kind = TyKind<Self>>;
-    type Tys: Clone + Debug + Hash + Ord + IntoIterator<Item = Self::Ty>;
-    type AliasTy: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type ParamTy: Clone + Debug + Hash + Ord;
-    type BoundTy: Clone + Debug + Hash + Ord;
-    type PlaceholderTy: Clone + Debug + Hash + Ord + Placeholder;
+    type Tys: Copy + Debug + Hash + Ord + IntoIterator<Item = Self::Ty>;
+    type AliasTy: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type ParamTy: Copy + Debug + Hash + Ord;
+    type BoundTy: Copy + Debug + Hash + Ord;
+    type PlaceholderTy: Copy + Debug + Hash + Ord + PlaceholderLike;
 
     // Things stored inside of tys
-    type ErrorGuaranteed: Clone + Debug + Hash + Ord;
-    type BoundExistentialPredicates: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type PolyFnSig: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type AllocId: Clone + Debug + Hash + Ord;
+    type ErrorGuaranteed: Copy + Debug + Hash + Ord;
+    type BoundExistentialPredicates: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type PolyFnSig: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type AllocId: Copy + Debug + Hash + Ord;
 
     // Kinds of consts
-    type Const: Clone
+    type Const: Copy
         + DebugWithInfcx<Self>
         + Hash
         + Ord
         + Into<Self::GenericArg>
         + IntoKind<Kind = ConstKind<Self>>
         + ConstTy<Self>;
-    type AliasConst: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type PlaceholderConst: Clone + Debug + Hash + Ord + Placeholder;
-    type ParamConst: Clone + Debug + Hash + Ord;
-    type BoundConst: Clone + Debug + Hash + Ord;
-    type ValueConst: Clone + Debug + Hash + Ord;
-    type ExprConst: Clone + DebugWithInfcx<Self> + Hash + Ord;
+    type AliasConst: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type PlaceholderConst: Copy + Debug + Hash + Ord + PlaceholderLike;
+    type ParamConst: Copy + Debug + Hash + Ord;
+    type BoundConst: Copy + Debug + Hash + Ord;
+    type ValueConst: Copy + Debug + Hash + Ord;
+    type ExprConst: Copy + DebugWithInfcx<Self> + Hash + Ord;
 
     // Kinds of regions
-    type Region: Clone
+    type Region: Copy
         + DebugWithInfcx<Self>
         + Hash
         + Ord
         + Into<Self::GenericArg>
         + IntoKind<Kind = RegionKind<Self>>;
-    type EarlyParamRegion: Clone + Debug + Hash + Ord;
-    type LateParamRegion: Clone + Debug + Hash + Ord;
-    type BoundRegion: Clone + Debug + Hash + Ord;
-    type InferRegion: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type PlaceholderRegion: Clone + Debug + Hash + Ord + Placeholder;
+    type EarlyParamRegion: Copy + Debug + Hash + Ord;
+    type LateParamRegion: Copy + Debug + Hash + Ord;
+    type BoundRegion: Copy + Debug + Hash + Ord;
+    type InferRegion: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type PlaceholderRegion: Copy + Debug + Hash + Ord + PlaceholderLike;
 
     // Predicates
-    type Predicate: Clone + Debug + Hash + Eq;
-    type TraitPredicate: Clone + Debug + Hash + Eq;
-    type RegionOutlivesPredicate: Clone + Debug + Hash + Eq;
-    type TypeOutlivesPredicate: Clone + Debug + Hash + Eq;
-    type ProjectionPredicate: Clone + Debug + Hash + Eq;
-    type NormalizesTo: Clone + Debug + Hash + Eq;
-    type SubtypePredicate: Clone + Debug + Hash + Eq;
-    type CoercePredicate: Clone + Debug + Hash + Eq;
-    type ClosureKind: Clone + Debug + Hash + Eq;
+    type Predicate: Copy + Debug + Hash + Eq;
+    type TraitPredicate: Copy + Debug + Hash + Eq;
+    type RegionOutlivesPredicate: Copy + Debug + Hash + Eq;
+    type TypeOutlivesPredicate: Copy + Debug + Hash + Eq;
+    type ProjectionPredicate: Copy + Debug + Hash + Eq;
+    type NormalizesTo: Copy + Debug + Hash + Eq;
+    type SubtypePredicate: Copy + Debug + Hash + Eq;
+    type CoercePredicate: Copy + Debug + Hash + Eq;
+    type ClosureKind: Copy + Debug + Hash + Eq;
 
     fn ty_and_mut_to_parts(ty_and_mut: Self::TypeAndMut) -> (Self::Ty, Mutability);
 
-    fn mk_canonical_var_infos(&self, infos: &[CanonicalVarInfo<Self>]) -> Self::CanonicalVars;
+    fn mk_canonical_var_infos(self, infos: &[CanonicalVarInfo<Self>]) -> Self::CanonicalVars;
 
-    fn mk_bound_ty(&self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Ty;
-    fn mk_bound_region(&self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Region;
-    fn mk_bound_const(&self, debruijn: DebruijnIndex, var: BoundVar, ty: Self::Ty) -> Self::Const;
+    // FIXME: We should not have all these constructors on `Interner`, but as functions on some trait.
+    fn mk_bound_ty(self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Ty;
+    fn mk_bound_region(self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Region;
+    fn mk_bound_const(self, debruijn: DebruijnIndex, var: BoundVar, ty: Self::Ty) -> Self::Const;
 }
 
 /// Common capabilities of placeholder kinds
-pub trait Placeholder {
-    fn universe(&self) -> UniverseIndex;
-    fn var(&self) -> BoundVar;
+pub trait PlaceholderLike {
+    fn universe(self) -> UniverseIndex;
+    fn var(self) -> BoundVar;
 
-    fn with_updated_universe(&self, ui: UniverseIndex) -> Self;
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self;
 
     fn new(ui: UniverseIndex, var: BoundVar) -> Self;
 }
@@ -104,11 +104,11 @@ pub trait Placeholder {
 pub trait IntoKind {
     type Kind;
 
-    fn kind(&self) -> Self::Kind;
+    fn kind(self) -> Self::Kind;
 }
 
 pub trait ConstTy<I: Interner> {
-    fn ty(&self) -> I::Ty;
+    fn ty(self) -> I::Ty;
 }
 
 /// Imagine you have a function `F: FnOnce(&[T]) -> R`, plus an iterator `iter`
diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs
index 1bcb1fcff35..200963ff7c5 100644
--- a/compiler/rustc_type_ir/src/lib.rs
+++ b/compiler/rustc_type_ir/src/lib.rs
@@ -4,6 +4,7 @@
 )]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
+#![allow(rustc::usage_of_ty_tykind)]
 #![cfg_attr(feature = "nightly", allow(internal_features))]
 
 #[cfg(feature = "nightly")]
@@ -52,6 +53,11 @@ pub use predicate_kind::*;
 pub use region_kind::*;
 pub use ty_info::*;
 pub use ty_kind::*;
+pub use AliasKind::*;
+pub use DynKind::*;
+pub use InferTy::*;
+pub use RegionKind::*;
+pub use TyKind::*;
 
 rustc_index::newtype_index! {
     /// A [De Bruijn index][dbi] is a standard means of representing
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index 9d3a578c3e4..a7a5cae254c 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -1,5 +1,3 @@
-#![allow(rustc::usage_of_ty_tykind)]
-
 #[cfg(feature = "nightly")]
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 #[cfg(feature = "nightly")]
@@ -394,7 +392,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
             Float(float) => write!(f, "{float:?}"),
             Adt(d, s) => {
                 write!(f, "{d:?}")?;
-                let mut s = s.clone().into_iter();
+                let mut s = s.into_iter();
                 let first = s.next();
                 match first {
                     Some(first) => write!(f, "<{:?}", first)?,
@@ -412,7 +410,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
             Array(t, c) => write!(f, "[{:?}; {:?}]", &this.wrap(t), &this.wrap(c)),
             Slice(t) => write!(f, "[{:?}]", &this.wrap(t)),
             RawPtr(p) => {
-                let (ty, mutbl) = I::ty_and_mut_to_parts(p.clone());
+                let (ty, mutbl) = I::ty_and_mut_to_parts(*p);
                 match mutbl {
                     Mutability::Mut => write!(f, "*mut "),
                     Mutability::Not => write!(f, "*const "),
@@ -442,7 +440,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
             Tuple(t) => {
                 write!(f, "(")?;
                 let mut count = 0;
-                for ty in t.clone() {
+                for ty in *t {
                     if count > 0 {
                         write!(f, ", ")?;
                     }
-- 
cgit 1.4.1-3-g733a5


From 80f240a539f3fa8c2892fccb0a0a54ca96589fc7 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Tue, 5 Dec 2023 18:58:14 +0000
Subject: Make it not depend on nightly conditionally

---
 compiler/rustc_next_trait_solver/Cargo.toml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_next_trait_solver/Cargo.toml b/compiler/rustc_next_trait_solver/Cargo.toml
index 88c4b62c54c..9d496fd8e81 100644
--- a/compiler/rustc_next_trait_solver/Cargo.toml
+++ b/compiler/rustc_next_trait_solver/Cargo.toml
@@ -4,4 +4,10 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-rustc_type_ir = { path = "../rustc_type_ir" }
+rustc_type_ir = { path = "../rustc_type_ir", default-features = false }
+
+[features]
+default = ["nightly"]
+nightly = [
+    "rustc_type_ir/nightly",
+]
\ No newline at end of file
-- 
cgit 1.4.1-3-g733a5


From 287c77e9212d6ad4e806c000eb6a4ee8e0ce462f Mon Sep 17 00:00:00 2001
From: Santiago Pastorino <spastorino@gmail.com>
Date: Fri, 1 Dec 2023 17:13:01 -0300
Subject: Add tests related to normalization in implied bounds

---
 tests/ui/implied-bounds/from-trait-impl.rs         | 24 ++++++++++++++++++
 .../normalization-nested.lifetime.stderr           | 29 ++++++++++++++++------
 tests/ui/implied-bounds/normalization-nested.rs    |  4 ++-
 ...normalization-preserve-equality.borrowck.stderr | 28 +++++++++++++++++++++
 .../normalization-preserve-equality.rs             | 28 +++++++++++++++++++++
 5 files changed, 105 insertions(+), 8 deletions(-)
 create mode 100644 tests/ui/implied-bounds/from-trait-impl.rs
 create mode 100644 tests/ui/implied-bounds/normalization-preserve-equality.borrowck.stderr
 create mode 100644 tests/ui/implied-bounds/normalization-preserve-equality.rs

diff --git a/tests/ui/implied-bounds/from-trait-impl.rs b/tests/ui/implied-bounds/from-trait-impl.rs
new file mode 100644
index 00000000000..d13fddd9b8d
--- /dev/null
+++ b/tests/ui/implied-bounds/from-trait-impl.rs
@@ -0,0 +1,24 @@
+// check-pass
+// known-bug: #109628
+
+trait Trait {
+    type Assoc;
+}
+
+impl<X: 'static> Trait for (X,) {
+    type Assoc = ();
+}
+
+struct Foo<T: Trait>(T)
+where
+    T::Assoc: Clone; // any predicate using `T::Assoc` works here
+
+fn func1(foo: Foo<(&str,)>) {
+    let _: &'static str = foo.0.0;
+}
+
+trait TestTrait {}
+
+impl<X> TestTrait for [Foo<(X,)>; 1] {}
+
+fn main() {}
diff --git a/tests/ui/implied-bounds/normalization-nested.lifetime.stderr b/tests/ui/implied-bounds/normalization-nested.lifetime.stderr
index abffee57a0f..e020230d86a 100644
--- a/tests/ui/implied-bounds/normalization-nested.lifetime.stderr
+++ b/tests/ui/implied-bounds/normalization-nested.lifetime.stderr
@@ -1,11 +1,11 @@
 error[E0759]: `fn` parameter has lifetime `'x` but it needs to satisfy a `'static` lifetime requirement
-  --> $DIR/normalization-nested.rs:35:20
+  --> $DIR/normalization-nested.rs:35:28
    |
-LL | pub fn test<'x>(_: Map<Vec<&'x ()>>, s: &'x str) -> &'static str {
-   |                    ^^^^^^^^^^^^^^^^
-   |                    |
-   |                    this data with lifetime `'x`...
-   |                    ...is used and required to live as long as `'static` here
+LL | pub fn test_wfcheck<'x>(_: Map<Vec<&'x ()>>) {}
+   |                            ^^^^^^^^^^^^^^^^
+   |                            |
+   |                            this data with lifetime `'x`...
+   |                            ...is used and required to live as long as `'static` here
    |
 note: `'static` lifetime requirement introduced by this bound
   --> $DIR/normalization-nested.rs:33:14
@@ -13,6 +13,21 @@ note: `'static` lifetime requirement introduced by this bound
 LL |     I::Item: 'static;
    |              ^^^^^^^
 
-error: aborting due to 1 previous error
+error[E0759]: `fn` parameter has lifetime `'x` but it needs to satisfy a `'static` lifetime requirement
+  --> $DIR/normalization-nested.rs:37:29
+   |
+LL | pub fn test_borrowck<'x>(_: Map<Vec<&'x ()>>, s: &'x str) -> &'static str {
+   |                             ^^^^^^^^^^^^^^^^
+   |                             |
+   |                             this data with lifetime `'x`...
+   |                             ...is used and required to live as long as `'static` here
+   |
+note: `'static` lifetime requirement introduced by this bound
+  --> $DIR/normalization-nested.rs:33:14
+   |
+LL |     I::Item: 'static;
+   |              ^^^^^^^
+
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0759`.
diff --git a/tests/ui/implied-bounds/normalization-nested.rs b/tests/ui/implied-bounds/normalization-nested.rs
index 5f1cbb3f697..87903783a67 100644
--- a/tests/ui/implied-bounds/normalization-nested.rs
+++ b/tests/ui/implied-bounds/normalization-nested.rs
@@ -32,7 +32,9 @@ where
     I: Iter,
     I::Item: 'static;
 
-pub fn test<'x>(_: Map<Vec<&'x ()>>, s: &'x str) -> &'static str {
+pub fn test_wfcheck<'x>(_: Map<Vec<&'x ()>>) {}
+
+pub fn test_borrowck<'x>(_: Map<Vec<&'x ()>>, s: &'x str) -> &'static str {
     s
 }
 
diff --git a/tests/ui/implied-bounds/normalization-preserve-equality.borrowck.stderr b/tests/ui/implied-bounds/normalization-preserve-equality.borrowck.stderr
new file mode 100644
index 00000000000..96c76ca9ac3
--- /dev/null
+++ b/tests/ui/implied-bounds/normalization-preserve-equality.borrowck.stderr
@@ -0,0 +1,28 @@
+error: lifetime may not live long enough
+  --> $DIR/normalization-preserve-equality.rs:24:1
+   |
+LL | fn test_borrowck<'a, 'b>(_: (<Equal<'a, 'b> as Trait>::Ty, Equal<'a, 'b>)) {
+   | ^^^^^^^^^^^^^^^^^--^^--^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | |                |   |
+   | |                |   lifetime `'b` defined here
+   | |                lifetime `'a` defined here
+   | requires that `'a` must outlive `'b`
+   |
+   = help: consider adding the following bound: `'a: 'b`
+
+error: lifetime may not live long enough
+  --> $DIR/normalization-preserve-equality.rs:24:1
+   |
+LL | fn test_borrowck<'a, 'b>(_: (<Equal<'a, 'b> as Trait>::Ty, Equal<'a, 'b>)) {
+   | ^^^^^^^^^^^^^^^^^--^^--^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | |                |   |
+   | |                |   lifetime `'b` defined here
+   | |                lifetime `'a` defined here
+   | requires that `'b` must outlive `'a`
+   |
+   = help: consider adding the following bound: `'b: 'a`
+
+help: `'a` and `'b` must be the same: replace one with the other
+
+error: aborting due to 2 previous errors
+
diff --git a/tests/ui/implied-bounds/normalization-preserve-equality.rs b/tests/ui/implied-bounds/normalization-preserve-equality.rs
new file mode 100644
index 00000000000..557c171e515
--- /dev/null
+++ b/tests/ui/implied-bounds/normalization-preserve-equality.rs
@@ -0,0 +1,28 @@
+// Both revisions should pass. `borrowck` revision is a bug!
+//
+// revisions: wfcheck borrowck
+// [wfcheck] check-pass
+// [borrowck] check-fail
+// [borrowck] known-bug: #106569
+
+struct Equal<'a, 'b>(&'a &'b (), &'b &'a ()); // implies 'a == 'b
+
+trait Trait {
+    type Ty;
+}
+
+impl<'x> Trait for Equal<'x, 'x> {
+    type Ty = ();
+}
+
+trait WfCheckTrait {}
+
+#[cfg(wfcheck)]
+impl<'a, 'b> WfCheckTrait for (<Equal<'a, 'b> as Trait>::Ty, Equal<'a, 'b>) {}
+
+#[cfg(borrowck)]
+fn test_borrowck<'a, 'b>(_: (<Equal<'a, 'b> as Trait>::Ty, Equal<'a, 'b>)) {
+    let _ = None::<Equal<'a, 'b>>;
+}
+
+fn main() {}
-- 
cgit 1.4.1-3-g733a5


From 8361a7288e42d303890a63092fe5fd276819c79d Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Fri, 8 Dec 2023 21:38:00 +0000
Subject: Introduce closure_id method on CoroutineKind

---
 compiler/rustc_ast/src/ast.rs               |  8 ++++++++
 compiler/rustc_ast_lowering/src/item.rs     |  5 +----
 compiler/rustc_lint/src/early.rs            | 10 ++--------
 compiler/rustc_resolve/src/def_collector.rs | 25 +++++++++++++------------
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 0b06af22365..190fae95652 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -2450,6 +2450,14 @@ impl CoroutineKind {
         matches!(self, CoroutineKind::Gen { .. })
     }
 
+    pub fn closure_id(self) -> NodeId {
+        match self {
+            CoroutineKind::Async { closure_id, .. }
+            | CoroutineKind::Gen { closure_id, .. }
+            | CoroutineKind::AsyncGen { closure_id, .. } => closure_id,
+        }
+    }
+
     /// In this case this is an `async` or `gen` return, the `NodeId` for the generated `impl Trait`
     /// item.
     pub fn return_id(self) -> (NodeId, Span) {
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 9d1f2684c39..a4effb99e71 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -1035,10 +1035,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let (Some(coroutine_kind), Some(body)) = (coroutine_kind, body) else {
             return self.lower_fn_body_block(span, decl, body);
         };
-        // FIXME(gen_blocks): Introduce `closure_id` method and remove ALL destructuring.
-        let (CoroutineKind::Async { closure_id, .. }
-        | CoroutineKind::Gen { closure_id, .. }
-        | CoroutineKind::AsyncGen { closure_id, .. }) = coroutine_kind;
+        let closure_id = coroutine_kind.closure_id();
 
         self.lower_body(|this| {
             let mut parameters: Vec<hir::Param<'_>> = Vec::new();
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index 80c6feaa269..71c9aa79e53 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -163,10 +163,7 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // it does not have a corresponding AST node
         if let ast_visit::FnKind::Fn(_, _, sig, _, _, _) = fk {
             if let Some(coro_kind) = sig.header.coroutine_kind {
-                let (ast::CoroutineKind::Async { closure_id, .. }
-                | ast::CoroutineKind::Gen { closure_id, .. }
-                | ast::CoroutineKind::AsyncGen { closure_id, .. }) = coro_kind;
-                self.check_id(closure_id);
+                self.check_id(coro_kind.closure_id());
             }
         }
     }
@@ -228,10 +225,7 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
             ast::ExprKind::Closure(box ast::Closure {
                 coroutine_kind: Some(coro_kind), ..
             }) => {
-                let (ast::CoroutineKind::Async { closure_id, .. }
-                | ast::CoroutineKind::Gen { closure_id, .. }
-                | ast::CoroutineKind::AsyncGen { closure_id, .. }) = coro_kind;
-                self.check_id(closure_id);
+                self.check_id(coro_kind.closure_id());
             }
             _ => {}
         }
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 186dd28b142..02553d50071 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -157,11 +157,7 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
     fn visit_fn(&mut self, fn_kind: FnKind<'a>, span: Span, _: NodeId) {
         if let FnKind::Fn(_, _, sig, _, generics, body) = fn_kind {
             match sig.header.coroutine_kind {
-                Some(
-                    CoroutineKind::Async { closure_id, .. }
-                    | CoroutineKind::Gen { closure_id, .. }
-                    | CoroutineKind::AsyncGen { closure_id, .. },
-                ) => {
+                Some(coroutine_kind) => {
                     self.visit_generics(generics);
 
                     // For async functions, we need to create their inner defs inside of a
@@ -176,8 +172,12 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
                     // then the closure_def will never be used, and we should avoid generating a
                     // def-id for it.
                     if let Some(body) = body {
-                        let closure_def =
-                            self.create_def(closure_id, kw::Empty, DefKind::Closure, span);
+                        let closure_def = self.create_def(
+                            coroutine_kind.closure_id(),
+                            kw::Empty,
+                            DefKind::Closure,
+                            span,
+                        );
                         self.with_parent(closure_def, |this| this.visit_block(body));
                     }
                     return;
@@ -289,11 +289,12 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
                 // we must create two defs.
                 let closure_def = self.create_def(expr.id, kw::Empty, DefKind::Closure, expr.span);
                 match closure.coroutine_kind {
-                    Some(
-                        CoroutineKind::Async { closure_id, .. }
-                        | CoroutineKind::Gen { closure_id, .. }
-                        | CoroutineKind::AsyncGen { closure_id, .. },
-                    ) => self.create_def(closure_id, kw::Empty, DefKind::Closure, expr.span),
+                    Some(coroutine_kind) => self.create_def(
+                        coroutine_kind.closure_id(),
+                        kw::Empty,
+                        DefKind::Closure,
+                        expr.span,
+                    ),
                     None => closure_def,
                 }
             }
-- 
cgit 1.4.1-3-g733a5


From d5dcd85376925f5f3d5fdbdcc9467165fe2c0b18 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Fri, 8 Dec 2023 21:46:08 +0000
Subject: More nits

---
 compiler/rustc_ast_lowering/src/lib.rs                               | 3 ++-
 .../rustc_trait_selection/src/traits/error_reporting/suggestions.rs  | 5 ++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 753650f7324..0c71165deed 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -177,7 +177,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             } else {
                 [sym::gen_future].into()
             },
-            // FIXME(gen_blocks): how does `closure_track_caller`
+            // FIXME(gen_blocks): how does `closure_track_caller`/`async_fn_track_caller`
+            // interact with `gen`/`async gen` blocks
             allow_async_iterator: [sym::gen_future, sym::async_iterator].into(),
             generics_def_id_map: Default::default(),
             host_param_id: None,
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index 7bf37cf7980..95ffd07e397 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -3144,10 +3144,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 let what = match self.tcx.coroutine_kind(coroutine_def_id) {
                     None
                     | Some(hir::CoroutineKind::Coroutine)
-                    | Some(hir::CoroutineKind::Gen(_))
-                    // FIXME(gen_blocks): This could be yield or await...
-                    | Some(hir::CoroutineKind::AsyncGen(_)) => "yield",
+                    | Some(hir::CoroutineKind::Gen(_)) => "yield",
                     Some(hir::CoroutineKind::Async(..)) => "await",
+                    Some(hir::CoroutineKind::AsyncGen(_)) => "yield`/`await",
                 };
                 err.note(format!(
                     "all values live across `{what}` must have a statically known size"
-- 
cgit 1.4.1-3-g733a5


From 384a49edd0cddba275e6be73acbb9a7c37ec03b2 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Fri, 8 Dec 2023 21:46:30 +0000
Subject: Rename some more coro_kind -> coroutine_kind

---
 compiler/rustc_ast_passes/src/ast_validation.rs | 4 ++--
 compiler/rustc_builtin_macros/src/test.rs       | 4 ++--
 compiler/rustc_lint/src/early.rs                | 9 +++++----
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 0644c4cd6be..1f9bc09f5f7 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -1271,11 +1271,11 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         // Functions cannot both be `const async` or `const gen`
         if let Some(&FnHeader {
             constness: Const::Yes(cspan),
-            coroutine_kind: Some(coro_kind),
+            coroutine_kind: Some(coroutine_kind),
             ..
         }) = fk.header()
         {
-            let aspan = match coro_kind {
+            let aspan = match coroutine_kind {
                 CoroutineKind::Async { span: aspan, .. }
                 | CoroutineKind::Gen { span: aspan, .. }
                 | CoroutineKind::AsyncGen { span: aspan, .. } => aspan,
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index 794be25955d..e5b274304e7 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -541,8 +541,8 @@ fn check_test_signature(
         return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "unsafe" }));
     }
 
-    if let Some(coro_kind) = f.sig.header.coroutine_kind {
-        match coro_kind {
+    if let Some(coroutine_kind) = f.sig.header.coroutine_kind {
+        match coroutine_kind {
             ast::CoroutineKind::Async { span, .. } => {
                 return Err(sd.emit_err(errors::TestBadFn {
                     span: i.span,
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index 71c9aa79e53..4c7f9eeff8c 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -162,8 +162,8 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // Explicitly check for lints associated with 'closure_id', since
         // it does not have a corresponding AST node
         if let ast_visit::FnKind::Fn(_, _, sig, _, _, _) = fk {
-            if let Some(coro_kind) = sig.header.coroutine_kind {
-                self.check_id(coro_kind.closure_id());
+            if let Some(coroutine_kind) = sig.header.coroutine_kind {
+                self.check_id(coroutine_kind.closure_id());
             }
         }
     }
@@ -223,9 +223,10 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // it does not have a corresponding AST node
         match e.kind {
             ast::ExprKind::Closure(box ast::Closure {
-                coroutine_kind: Some(coro_kind), ..
+                coroutine_kind: Some(coroutine_kind),
+                ..
             }) => {
-                self.check_id(coro_kind.closure_id());
+                self.check_id(coroutine_kind.closure_id());
             }
             _ => {}
         }
-- 
cgit 1.4.1-3-g733a5


From e9878125216e660601182dcadbc1c86910333a64 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Fri, 8 Dec 2023 22:25:12 +0000
Subject: Make async generators fused by default

---
 compiler/rustc_mir_transform/src/coroutine.rs | 44 ++++++++++++++++++++-------
 tests/ui/coroutine/async_gen_fn_iter.rs       |  4 +++
 2 files changed, 37 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index 2b591abb05d..737fb6bf612 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -252,15 +252,15 @@ struct TransformVisitor<'tcx> {
 
 impl<'tcx> TransformVisitor<'tcx> {
     fn insert_none_ret_block(&self, body: &mut Body<'tcx>) -> BasicBlock {
-        assert!(matches!(self.coroutine_kind, CoroutineKind::Gen(_)));
-
         let block = BasicBlock::new(body.basic_blocks.len());
         let source_info = SourceInfo::outermost(body.span);
-        let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
 
-        let statements = vec![Statement {
-            kind: StatementKind::Assign(Box::new((
-                Place::return_place(),
+        let none_value = match self.coroutine_kind {
+            CoroutineKind::Async(_) => span_bug!(body.span, "`Future`s are not fused inherently"),
+            CoroutineKind::Coroutine => span_bug!(body.span, "`Coroutine`s cannot be fused"),
+            // `gen` continues return `None`
+            CoroutineKind::Gen(_) => {
+                let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
                 Rvalue::Aggregate(
                     Box::new(AggregateKind::Adt(
                         option_def_id,
@@ -270,8 +270,29 @@ impl<'tcx> TransformVisitor<'tcx> {
                         None,
                     )),
                     IndexVec::new(),
-                ),
-            ))),
+                )
+            }
+            // `async gen` continues to return `Poll::Ready(None)`
+            CoroutineKind::AsyncGen(_) => {
+                let ty::Adt(_poll_adt, args) = *self.old_yield_ty.kind() else { bug!() };
+                let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else { bug!() };
+                let yield_ty = args.type_at(0);
+                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                    span: source_info.span,
+                    const_: Const::Unevaluated(
+                        UnevaluatedConst::new(
+                            self.tcx.require_lang_item(LangItem::AsyncGenFinished, None),
+                            self.tcx.mk_args(&[yield_ty.into()]),
+                        ),
+                        self.old_yield_ty,
+                    ),
+                    user_ty: None,
+                })))
+            }
+        };
+
+        let statements = vec![Statement {
+            kind: StatementKind::Assign(Box::new((Place::return_place(), none_value))),
             source_info,
         }];
 
@@ -1393,11 +1414,12 @@ fn create_coroutine_resume_function<'tcx>(
 
     if can_return {
         let block = match coroutine_kind {
-            // FIXME(gen_blocks): Should `async gen` yield `None` when resumed once again?
-            CoroutineKind::Async(_) | CoroutineKind::AsyncGen(_) | CoroutineKind::Coroutine => {
+            CoroutineKind::Async(_) | CoroutineKind::Coroutine => {
                 insert_panic_block(tcx, body, ResumedAfterReturn(coroutine_kind))
             }
-            CoroutineKind::Gen(_) => transform.insert_none_ret_block(body),
+            CoroutineKind::AsyncGen(_) | CoroutineKind::Gen(_) => {
+                transform.insert_none_ret_block(body)
+            }
         };
         cases.insert(1, (RETURNED, block));
     }
diff --git a/tests/ui/coroutine/async_gen_fn_iter.rs b/tests/ui/coroutine/async_gen_fn_iter.rs
index 6f8f3feb87e..4fa29e1095a 100644
--- a/tests/ui/coroutine/async_gen_fn_iter.rs
+++ b/tests/ui/coroutine/async_gen_fn_iter.rs
@@ -33,6 +33,10 @@ async fn async_main() {
     assert_eq!(iter.as_mut().next().await, Some(2));
     assert_eq!(iter.as_mut().next().await, Some(3));
     assert_eq!(iter.as_mut().next().await, None);
+
+    // Test that the iterator is fused and does not panic
+    assert_eq!(iter.as_mut().next().await, None);
+    assert_eq!(iter.as_mut().next().await, None);
 }
 
 // ------------------------------------------------------------------------- //
-- 
cgit 1.4.1-3-g733a5


From 08b8ba0a3207b2c5e34cab7f85323908f8bd8b37 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 6 Dec 2023 20:43:59 +1100
Subject: Add some useful comments.

---
 compiler/rustc_lexer/src/unescape.rs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index dab656b35f9..5559c03edea 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -7,7 +7,9 @@ use std::str::Chars;
 #[cfg(test)]
 mod tests;
 
-/// Errors and warnings that can occur during string unescaping.
+/// Errors and warnings that can occur during string unescaping. They mostly
+/// relate to malformed escape sequences, but there are a few that are about
+/// other problems.
 #[derive(Debug, PartialEq, Eq)]
 pub enum EscapeError {
     /// Expected 1 char, but 0 were found.
@@ -73,9 +75,11 @@ impl EscapeError {
     }
 }
 
-/// Takes a contents of a literal (without quotes) and produces a
-/// sequence of escaped characters or errors.
-/// Values are returned through invoking of the provided callback.
+/// Takes a contents of a literal (without quotes) and produces a sequence of
+/// escaped characters or errors.
+///
+/// Values are returned by invoking `callback`. For `Char` and `Byte` modes,
+/// the callback will be called exactly once.
 pub fn unescape_literal<F>(src: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
-- 
cgit 1.4.1-3-g733a5


From f883762970b4831e61b7b3355981003868946897 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 7 Dec 2023 09:16:18 +1100
Subject: Remove explicit `\n` and `\t` handling in `unescape_str_common`.

The fallback `_` case works for these chars, no need to treat them
specially.
---
 compiler/rustc_lexer/src/unescape.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 5559c03edea..545c579cdb5 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -351,8 +351,6 @@ where
                     _ => scan_escape::<T>(&mut chars, mode),
                 }
             }
-            '\n' => Ok(b'\n'.into()),
-            '\t' => Ok(b'\t'.into()),
             '"' => Err(EscapeError::EscapeOnlyChar),
             '\r' => Err(EscapeError::BareCarriageReturn),
             _ => ascii_check(c, mode.characters_should_be_ascii()).map(Into::into),
-- 
cgit 1.4.1-3-g733a5


From 119b1d0c63760eb309da5176f31a58f9854d6815 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 7 Dec 2023 09:25:53 +1100
Subject: Eliminate `is_byte: bool` args in unescaping code.

These don't really make sense since C string literals were added. This
commit removes them in favour for `mode: Mode` args. `ascii_check` still
has a `characters_should_be_ascii: bool` arg.

Also, `characters_should_be_ascii` is renamed to be shorter.
---
 compiler/rustc_lexer/src/unescape.rs | 40 +++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 545c579cdb5..8c5b2e3635e 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -87,14 +87,12 @@ where
     match mode {
         Mode::Char | Mode::Byte => {
             let mut chars = src.chars();
-            let res = unescape_char_or_byte(&mut chars, mode == Mode::Byte);
+            let res = unescape_char_or_byte(&mut chars, mode);
             callback(0..(src.len() - chars.as_str().len()), res);
         }
         Mode::Str | Mode::ByteStr => unescape_str_common(src, mode, callback),
 
-        Mode::RawStr | Mode::RawByteStr => {
-            unescape_raw_str_or_raw_byte_str(src, mode == Mode::RawByteStr, callback)
-        }
+        Mode::RawStr | Mode::RawByteStr => unescape_raw_str_or_raw_byte_str(src, mode, callback),
         Mode::CStr | Mode::RawCStr => unreachable!(),
     }
 }
@@ -122,11 +120,9 @@ where
     F: FnMut(Range<usize>, Result<CStrUnit, EscapeError>),
 {
     if mode == Mode::RawCStr {
-        unescape_raw_str_or_raw_byte_str(
-            src,
-            mode.characters_should_be_ascii(),
-            &mut |r, result| callback(r, result.map(CStrUnit::Char)),
-        );
+        unescape_raw_str_or_raw_byte_str(src, mode, &mut |r, result| {
+            callback(r, result.map(CStrUnit::Char))
+        });
     } else {
         unescape_str_common(src, mode, callback);
     }
@@ -135,13 +131,13 @@ where
 /// Takes a contents of a char literal (without quotes), and returns an
 /// unescaped char or an error.
 pub fn unescape_char(src: &str) -> Result<char, EscapeError> {
-    unescape_char_or_byte(&mut src.chars(), false)
+    unescape_char_or_byte(&mut src.chars(), Mode::Char)
 }
 
 /// Takes a contents of a byte literal (without quotes), and returns an
 /// unescaped byte or an error.
 pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> {
-    unescape_char_or_byte(&mut src.chars(), true).map(byte_from_char)
+    unescape_char_or_byte(&mut src.chars(), Mode::Byte).map(byte_from_char)
 }
 
 /// What kind of literal do we parse.
@@ -180,7 +176,8 @@ impl Mode {
     }
 
     /// Whether characters within the literal must be within the ASCII range
-    fn characters_should_be_ascii(self) -> bool {
+    #[inline]
+    fn chars_should_be_ascii(self) -> bool {
         match self {
             Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
             Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false,
@@ -299,22 +296,21 @@ fn scan_unicode(
 }
 
 #[inline]
-fn ascii_check(c: char, characters_should_be_ascii: bool) -> Result<char, EscapeError> {
-    if characters_should_be_ascii && !c.is_ascii() {
-        // Byte literal can't be a non-ascii character.
+fn ascii_check(c: char, chars_should_be_ascii: bool) -> Result<char, EscapeError> {
+    if chars_should_be_ascii && !c.is_ascii() {
         Err(EscapeError::NonAsciiCharInByte)
     } else {
         Ok(c)
     }
 }
 
-fn unescape_char_or_byte(chars: &mut Chars<'_>, is_byte: bool) -> Result<char, EscapeError> {
+fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
     let c = chars.next().ok_or(EscapeError::ZeroChars)?;
     let res = match c {
-        '\\' => scan_escape(chars, if is_byte { Mode::Byte } else { Mode::Char }),
+        '\\' => scan_escape(chars, mode),
         '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
         '\r' => Err(EscapeError::BareCarriageReturn),
-        _ => ascii_check(c, is_byte),
+        _ => ascii_check(c, mode.chars_should_be_ascii()),
     }?;
     if chars.next().is_some() {
         return Err(EscapeError::MoreThanOneChar);
@@ -329,6 +325,7 @@ where
     F: FnMut(Range<usize>, Result<T, EscapeError>),
 {
     let mut chars = src.chars();
+    let chars_should_be_ascii = mode.chars_should_be_ascii(); // get this outside the loop
 
     // The `start` and `end` computation here is complicated because
     // `skip_ascii_whitespace` makes us to skip over chars without counting
@@ -353,7 +350,7 @@ where
             }
             '"' => Err(EscapeError::EscapeOnlyChar),
             '\r' => Err(EscapeError::BareCarriageReturn),
-            _ => ascii_check(c, mode.characters_should_be_ascii()).map(Into::into),
+            _ => ascii_check(c, chars_should_be_ascii).map(Into::into),
         };
         let end = src.len() - chars.as_str().len();
         callback(start..end, res.map(Into::into));
@@ -390,11 +387,12 @@ where
 /// sequence of characters or errors.
 /// NOTE: Raw strings do not perform any explicit character escaping, here we
 /// only produce errors on bare CR.
-fn unescape_raw_str_or_raw_byte_str<F>(src: &str, is_byte: bool, callback: &mut F)
+fn unescape_raw_str_or_raw_byte_str<F>(src: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
 {
     let mut chars = src.chars();
+    let chars_should_be_ascii = mode.chars_should_be_ascii(); // get this outside the loop
 
     // The `start` and `end` computation here matches the one in
     // `unescape_str_common` for consistency, even though this function
@@ -403,7 +401,7 @@ where
         let start = src.len() - chars.as_str().len() - c.len_utf8();
         let res = match c {
             '\r' => Err(EscapeError::BareCarriageReturnInRawString),
-            _ => ascii_check(c, is_byte),
+            _ => ascii_check(c, chars_should_be_ascii),
         };
         let end = src.len() - chars.as_str().len();
         callback(start..end, res);
-- 
cgit 1.4.1-3-g733a5


From 9741dba7fa3fd61a8851e906861ac88bc527ca4c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 7 Dec 2023 10:49:42 +1100
Subject: Tweak the `no-nuls.rs` test.

The `empty!` macro calls should be outside the `cfg(FALSE)` function.
---
 tests/ui/rfcs/rfc-3348-c-string-literals/no-nuls.rs | Bin 594 -> 738 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/tests/ui/rfcs/rfc-3348-c-string-literals/no-nuls.rs b/tests/ui/rfcs/rfc-3348-c-string-literals/no-nuls.rs
index a7e36b2233e..e20ca50b88f 100644
Binary files a/tests/ui/rfcs/rfc-3348-c-string-literals/no-nuls.rs and b/tests/ui/rfcs/rfc-3348-c-string-literals/no-nuls.rs differ
-- 
cgit 1.4.1-3-g733a5


From adc46e5c08df7d3fe68293ee94add19b653b20aa Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 7 Dec 2023 18:52:11 +1100
Subject: Remove an unnecessary `into`.

---
 compiler/rustc_lexer/src/unescape.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 8c5b2e3635e..72cb78f82a2 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -353,7 +353,7 @@ where
             _ => ascii_check(c, chars_should_be_ascii).map(Into::into),
         };
         let end = src.len() - chars.as_str().len();
-        callback(start..end, res.map(Into::into));
+        callback(start..end, res);
     }
 }
 
-- 
cgit 1.4.1-3-g733a5


From 0a401b624bff7cfa776bac2f767b21ffdb1bd98f Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 7 Dec 2023 11:18:09 +1100
Subject: Tweak `Mode`.

- Add `use Mode::*` to avoid all the qualifiers.
- Reorder the variants. The existing order makes no particular sense,
  which has bugged me for some time. I've chosen an order that makes
  sense to me.
---
 compiler/rustc_lexer/src/unescape.rs | 72 +++++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 34 deletions(-)

diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 72cb78f82a2..249126a269e 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -4,6 +4,8 @@
 use std::ops::Range;
 use std::str::Chars;
 
+use Mode::*;
+
 #[cfg(test)]
 mod tests;
 
@@ -85,15 +87,14 @@ where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
 {
     match mode {
-        Mode::Char | Mode::Byte => {
+        Char | Byte => {
             let mut chars = src.chars();
             let res = unescape_char_or_byte(&mut chars, mode);
             callback(0..(src.len() - chars.as_str().len()), res);
         }
-        Mode::Str | Mode::ByteStr => unescape_str_common(src, mode, callback),
-
-        Mode::RawStr | Mode::RawByteStr => unescape_raw_str_or_raw_byte_str(src, mode, callback),
-        Mode::CStr | Mode::RawCStr => unreachable!(),
+        Str | ByteStr => unescape_str_common(src, mode, callback),
+        RawStr | RawByteStr => unescape_raw_str_or_raw_byte_str(src, mode, callback),
+        CStr | RawCStr => unreachable!(),
     }
 }
 
@@ -119,36 +120,44 @@ pub fn unescape_c_string<F>(src: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<CStrUnit, EscapeError>),
 {
-    if mode == Mode::RawCStr {
-        unescape_raw_str_or_raw_byte_str(src, mode, &mut |r, result| {
-            callback(r, result.map(CStrUnit::Char))
-        });
-    } else {
-        unescape_str_common(src, mode, callback);
+    match mode {
+        CStr => {
+            unescape_str_common(src, mode, callback);
+        }
+        RawCStr => {
+            unescape_raw_str_or_raw_byte_str(src, mode, &mut |r, result| {
+                callback(r, result.map(CStrUnit::Char))
+            });
+        }
+        Char | Byte | Str | RawStr | ByteStr | RawByteStr => unreachable!(),
     }
 }
 
 /// Takes a contents of a char literal (without quotes), and returns an
 /// unescaped char or an error.
 pub fn unescape_char(src: &str) -> Result<char, EscapeError> {
-    unescape_char_or_byte(&mut src.chars(), Mode::Char)
+    unescape_char_or_byte(&mut src.chars(), Char)
 }
 
 /// Takes a contents of a byte literal (without quotes), and returns an
 /// unescaped byte or an error.
 pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> {
-    unescape_char_or_byte(&mut src.chars(), Mode::Byte).map(byte_from_char)
+    unescape_char_or_byte(&mut src.chars(), Byte).map(byte_from_char)
 }
 
 /// What kind of literal do we parse.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum Mode {
     Char,
-    Str,
+
     Byte,
-    ByteStr,
+
+    Str,
     RawStr,
+
+    ByteStr,
     RawByteStr,
+
     CStr,
     RawCStr,
 }
@@ -156,47 +165,42 @@ pub enum Mode {
 impl Mode {
     pub fn in_double_quotes(self) -> bool {
         match self {
-            Mode::Str
-            | Mode::ByteStr
-            | Mode::RawStr
-            | Mode::RawByteStr
-            | Mode::CStr
-            | Mode::RawCStr => true,
-            Mode::Char | Mode::Byte => false,
+            Str | RawStr | ByteStr | RawByteStr | CStr | RawCStr => true,
+            Char | Byte => false,
         }
     }
 
     /// Non-byte literals should have `\xXX` escapes that are within the ASCII range.
     fn ascii_escapes_should_be_ascii(self) -> bool {
         match self {
-            Mode::Char | Mode::Str => true,
-            Mode::Byte | Mode::ByteStr | Mode::CStr => false,
-            Mode::RawStr | Mode::RawByteStr | Mode::RawCStr => unreachable!(),
+            Char | Str => true,
+            Byte | ByteStr | CStr => false,
+            RawStr | RawByteStr | RawCStr => unreachable!(),
         }
     }
 
-    /// Whether characters within the literal must be within the ASCII range
+    /// Whether characters within the literal must be within the ASCII range.
     #[inline]
     fn chars_should_be_ascii(self) -> bool {
         match self {
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
-            Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false,
+            Byte | ByteStr | RawByteStr => true,
+            Char | Str | RawStr | CStr | RawCStr => false,
         }
     }
 
     /// Byte literals do not allow unicode escape.
     fn is_unicode_escape_disallowed(self) -> bool {
         match self {
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
-            Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false,
+            Byte | ByteStr | RawByteStr => true,
+            Char | Str | RawStr | CStr | RawCStr => false,
         }
     }
 
     pub fn prefix_noraw(self) -> &'static str {
         match self {
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr => "b",
-            Mode::CStr | Mode::RawCStr => "c",
-            Mode::Char | Mode::Str | Mode::RawStr => "",
+            Char | Str | RawStr => "",
+            Byte | ByteStr | RawByteStr => "b",
+            CStr | RawCStr => "c",
         }
     }
 }
@@ -411,7 +415,7 @@ where
 #[inline]
 pub fn byte_from_char(c: char) -> u8 {
     let res = c as u32;
-    debug_assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr");
+    debug_assert!(res <= u8::MAX as u32, "guaranteed because of ByteStr");
     res as u8
 }
 
-- 
cgit 1.4.1-3-g733a5


From 7079adb226ea27a573715da6a789e8c5cb2609d9 Mon Sep 17 00:00:00 2001
From: Santiago Pastorino <spastorino@gmail.com>
Date: Fri, 8 Dec 2023 18:20:14 -0300
Subject: Add Bevy related test cases

---
 tests/ui/implied-bounds/auxiliary/bevy_ecs.rs | 18 +++++++++++++
 tests/ui/implied-bounds/bevy_world_query.rs   | 11 ++++++++
 tests/ui/implied-bounds/gluon_salsa.rs        | 31 ++++++++++++++++++++++
 tests/ui/implied-bounds/sod_service_chain.rs  | 37 +++++++++++++++++++++++++++
 4 files changed, 97 insertions(+)
 create mode 100644 tests/ui/implied-bounds/auxiliary/bevy_ecs.rs
 create mode 100644 tests/ui/implied-bounds/bevy_world_query.rs
 create mode 100644 tests/ui/implied-bounds/gluon_salsa.rs
 create mode 100644 tests/ui/implied-bounds/sod_service_chain.rs

diff --git a/tests/ui/implied-bounds/auxiliary/bevy_ecs.rs b/tests/ui/implied-bounds/auxiliary/bevy_ecs.rs
new file mode 100644
index 00000000000..b373d39f4d9
--- /dev/null
+++ b/tests/ui/implied-bounds/auxiliary/bevy_ecs.rs
@@ -0,0 +1,18 @@
+// Related to Bevy regression #118553
+
+pub trait WorldQuery {}
+impl WorldQuery for &u8 {}
+
+pub struct Query<Q: WorldQuery>(Q);
+
+pub trait SystemParam {
+    type State;
+}
+impl<Q: WorldQuery + 'static> SystemParam for Query<Q> {
+    type State = ();
+    // `Q: 'static` is required because we need the TypeId of Q ...
+}
+
+pub struct ParamSet<T: SystemParam>(T)
+where
+    T::State: Sized;
diff --git a/tests/ui/implied-bounds/bevy_world_query.rs b/tests/ui/implied-bounds/bevy_world_query.rs
new file mode 100644
index 00000000000..f8e64632676
--- /dev/null
+++ b/tests/ui/implied-bounds/bevy_world_query.rs
@@ -0,0 +1,11 @@
+// aux-crate:bevy_ecs=bevy_ecs.rs
+// check-pass
+// Related to Bevy regression #118553
+
+extern crate bevy_ecs;
+
+use bevy_ecs::*;
+
+fn handler<'a>(_: ParamSet<Query<&'a u8>>) {}
+
+fn main() {}
diff --git a/tests/ui/implied-bounds/gluon_salsa.rs b/tests/ui/implied-bounds/gluon_salsa.rs
new file mode 100644
index 00000000000..98951af8ac2
--- /dev/null
+++ b/tests/ui/implied-bounds/gluon_salsa.rs
@@ -0,0 +1,31 @@
+// check-pass
+// Related to Bevy regression #118553
+
+pub trait QueryBase {
+    type Db;
+}
+
+pub trait AsyncQueryFunction<'f>: // 'f is important
+    QueryBase<Db = <Self as AsyncQueryFunction<'f>>::SendDb> // bound is important
+{
+    type SendDb;
+}
+
+pub struct QueryTable<'me, Q, DB> {
+    _q: Option<Q>,
+    _db: Option<DB>,
+    _marker: Option<&'me ()>,
+}
+
+impl<'me, Q> QueryTable<'me, Q, <Q as QueryBase>::Db>
+// projection is important
+//   ^^^ removing 'me (and in QueryTable) gives a different error
+where
+    Q: for<'f> AsyncQueryFunction<'f>,
+{
+    pub fn get_async<'a>(&'a mut self) {
+        panic!();
+    }
+}
+
+fn main() {}
diff --git a/tests/ui/implied-bounds/sod_service_chain.rs b/tests/ui/implied-bounds/sod_service_chain.rs
new file mode 100644
index 00000000000..f45ced71f75
--- /dev/null
+++ b/tests/ui/implied-bounds/sod_service_chain.rs
@@ -0,0 +1,37 @@
+// check-pass
+// Related to crater regressions on #118553
+
+pub trait Debug {}
+
+pub trait Service {
+    type Input;
+    type Output;
+    type Error;
+}
+
+pub struct ServiceChain<P, S> {
+    prev: P,
+    service: S,
+}
+impl<P: Service, S: Service<Input = P::Output>> Service for ServiceChain<P, S>
+where
+    P::Error: 'static,
+    S::Error: 'static,
+{
+    type Input = P::Input;
+    type Output = S::Output;
+    type Error = ();
+}
+
+pub struct ServiceChainBuilder<P: Service, S: Service<Input = P::Output>> {
+    chain: ServiceChain<P, S>,
+}
+impl<P: Service, S: Service<Input = P::Output>> ServiceChainBuilder<P, S> {
+    pub fn next<NS: Service<Input = S::Output>>(
+        self,
+    ) -> ServiceChainBuilder<ServiceChain<P, S>, NS> {
+        panic!();
+    }
+}
+
+fn main() {}
-- 
cgit 1.4.1-3-g733a5


From 4e376cc104bc45e9dcac04374e21b5e8b976d282 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sun, 29 Oct 2023 10:13:07 +0100
Subject: Test empty types better

---
 ...ty-match-check-notes.exhaustive_patterns.stderr |  63 ++
 .../empty-match-check-notes.normal.stderr          |  62 ++
 .../pattern/usefulness/empty-match-check-notes.rs  |  52 ++
 .../empty-match.exhaustive_patterns.stderr         | 284 +++-----
 .../pattern/usefulness/empty-match.normal.stderr   | 283 +++-----
 tests/ui/pattern/usefulness/empty-match.rs         | 178 +----
 .../empty-types.exhaustive_patterns.stderr         | 776 +++++++++++++++++++++
 .../pattern/usefulness/empty-types.normal.stderr   | 721 +++++++++++++++++++
 tests/ui/pattern/usefulness/empty-types.rs         | 665 ++++++++++++++++++
 tests/ui/uninhabited/uninhabited-patterns.rs       |  12 +-
 tests/ui/uninhabited/uninhabited-patterns.stderr   |  12 +-
 11 files changed, 2606 insertions(+), 502 deletions(-)
 create mode 100644 tests/ui/pattern/usefulness/empty-match-check-notes.exhaustive_patterns.stderr
 create mode 100644 tests/ui/pattern/usefulness/empty-match-check-notes.normal.stderr
 create mode 100644 tests/ui/pattern/usefulness/empty-match-check-notes.rs
 create mode 100644 tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr
 create mode 100644 tests/ui/pattern/usefulness/empty-types.normal.stderr
 create mode 100644 tests/ui/pattern/usefulness/empty-types.rs

diff --git a/tests/ui/pattern/usefulness/empty-match-check-notes.exhaustive_patterns.stderr b/tests/ui/pattern/usefulness/empty-match-check-notes.exhaustive_patterns.stderr
new file mode 100644
index 00000000000..304435cb21e
--- /dev/null
+++ b/tests/ui/pattern/usefulness/empty-match-check-notes.exhaustive_patterns.stderr
@@ -0,0 +1,63 @@
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:17:9
+   |
+LL |         _ => {}
+   |         ^
+   |
+note: the lint level is defined here
+  --> $DIR/empty-match-check-notes.rs:7:9
+   |
+LL | #![deny(unreachable_patterns)]
+   |         ^^^^^^^^^^^^^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:20:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:27:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:30:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error[E0005]: refutable pattern in local binding
+  --> $DIR/empty-match-check-notes.rs:35:9
+   |
+LL |     let None = x;
+   |         ^^^^ pattern `Some(_)` not covered
+   |
+   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
+   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
+   = note: pattern `Some(_)` is currently uninhabited, but this variant contains private fields which may become inhabited in the future
+   = note: the matched value is of type `Option<SecretlyUninhabitedForeignStruct>`
+help: you might want to use `if let` to ignore the variant that isn't matched
+   |
+LL |     if let None = x { todo!() };
+   |     ++              +++++++++++
+
+error[E0004]: non-exhaustive patterns: `_` not covered
+  --> $DIR/empty-match-check-notes.rs:45:11
+   |
+LL |     match 0u8 {
+   |           ^^^ pattern `_` not covered
+   |
+   = note: the matched value is of type `u8`
+   = note: match arms with guards don't count towards exhaustivity
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         _ if false => {},
+LL +         _ => todo!()
+   |
+
+error: aborting due to 6 previous errors
+
+Some errors have detailed explanations: E0004, E0005.
+For more information about an error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-match-check-notes.normal.stderr b/tests/ui/pattern/usefulness/empty-match-check-notes.normal.stderr
new file mode 100644
index 00000000000..40494b726f0
--- /dev/null
+++ b/tests/ui/pattern/usefulness/empty-match-check-notes.normal.stderr
@@ -0,0 +1,62 @@
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:17:9
+   |
+LL |         _ => {}
+   |         ^
+   |
+note: the lint level is defined here
+  --> $DIR/empty-match-check-notes.rs:7:9
+   |
+LL | #![deny(unreachable_patterns)]
+   |         ^^^^^^^^^^^^^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:20:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:27:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-match-check-notes.rs:30:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error[E0005]: refutable pattern in local binding
+  --> $DIR/empty-match-check-notes.rs:35:9
+   |
+LL |     let None = x;
+   |         ^^^^ pattern `Some(_)` not covered
+   |
+   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
+   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
+   = note: the matched value is of type `Option<SecretlyUninhabitedForeignStruct>`
+help: you might want to use `if let` to ignore the variant that isn't matched
+   |
+LL |     if let None = x { todo!() };
+   |     ++              +++++++++++
+
+error[E0004]: non-exhaustive patterns: `_` not covered
+  --> $DIR/empty-match-check-notes.rs:45:11
+   |
+LL |     match 0u8 {
+   |           ^^^ pattern `_` not covered
+   |
+   = note: the matched value is of type `u8`
+   = note: match arms with guards don't count towards exhaustivity
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         _ if false => {},
+LL +         _ => todo!()
+   |
+
+error: aborting due to 6 previous errors
+
+Some errors have detailed explanations: E0004, E0005.
+For more information about an error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-match-check-notes.rs b/tests/ui/pattern/usefulness/empty-match-check-notes.rs
new file mode 100644
index 00000000000..ee9ff3dcf90
--- /dev/null
+++ b/tests/ui/pattern/usefulness/empty-match-check-notes.rs
@@ -0,0 +1,52 @@
+// aux-build:empty.rs
+// revisions: normal exhaustive_patterns
+//
+// This tests a match with no arms on various types, and checks NOTEs.
+#![feature(never_type)]
+#![cfg_attr(exhaustive_patterns, feature(exhaustive_patterns))]
+#![deny(unreachable_patterns)]
+//~^ NOTE the lint level is defined here
+
+extern crate empty;
+
+enum EmptyEnum {}
+
+fn empty_enum(x: EmptyEnum) {
+    match x {} // ok
+    match x {
+        _ => {} //~ ERROR unreachable pattern
+    }
+    match x {
+        _ if false => {} //~ ERROR unreachable pattern
+    }
+}
+
+fn empty_foreign_enum(x: empty::EmptyForeignEnum) {
+    match x {} // ok
+    match x {
+        _ => {} //~ ERROR unreachable pattern
+    }
+    match x {
+        _ if false => {} //~ ERROR unreachable pattern
+    }
+}
+
+fn empty_foreign_enum_private(x: Option<empty::SecretlyUninhabitedForeignStruct>) {
+    let None = x;
+    //~^ ERROR refutable pattern in local binding
+    //~| NOTE `let` bindings require an "irrefutable pattern"
+    //~| NOTE for more information, visit
+    //~| NOTE the matched value is of type
+    //~| NOTE pattern `Some(_)` not covered
+    //[exhaustive_patterns]~| NOTE currently uninhabited, but this variant contains private fields
+}
+
+fn main() {
+    match 0u8 {
+        //~^ ERROR `_` not covered
+        //~| NOTE the matched value is of type
+        //~| NOTE match arms with guards don't count towards exhaustivity
+        //~| NOTE pattern `_` not covered
+        _ if false => {}
+    }
+}
diff --git a/tests/ui/pattern/usefulness/empty-match.exhaustive_patterns.stderr b/tests/ui/pattern/usefulness/empty-match.exhaustive_patterns.stderr
index 8f9bd5bde89..9c3bebd7797 100644
--- a/tests/ui/pattern/usefulness/empty-match.exhaustive_patterns.stderr
+++ b/tests/ui/pattern/usefulness/empty-match.exhaustive_patterns.stderr
@@ -1,62 +1,5 @@
-error: unreachable pattern
-  --> $DIR/empty-match.rs:68:9
-   |
-LL |         _ => {},
-   |         ^
-   |
-note: the lint level is defined here
-  --> $DIR/empty-match.rs:8:9
-   |
-LL | #![deny(unreachable_patterns)]
-   |         ^^^^^^^^^^^^^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:71:9
-   |
-LL |         _ if false => {},
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:78:9
-   |
-LL |         _ => {},
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:81:9
-   |
-LL |         _ if false => {},
-   |         ^
-
-error[E0005]: refutable pattern in local binding
-  --> $DIR/empty-match.rs:86:9
-   |
-LL |     let None = x;
-   |         ^^^^ pattern `Some(_)` not covered
-   |
-   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
-   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
-   = note: pattern `Some(_)` is currently uninhabited, but this variant contains private fields which may become inhabited in the future
-   = note: the matched value is of type `Option<SecretlyUninhabitedForeignStruct>`
-help: you might want to use `if let` to ignore the variant that isn't matched
-   |
-LL |     if let None = x { todo!() };
-   |     ++              +++++++++++
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:98:9
-   |
-LL |         _ => {},
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:101:9
-   |
-LL |         _ if false => {},
-   |         ^
-
 error[E0004]: non-exhaustive patterns: type `u8` is non-empty
-  --> $DIR/empty-match.rs:119:20
+  --> $DIR/empty-match.rs:46:20
    |
 LL |     match_no_arms!(0u8);
    |                    ^^^
@@ -65,122 +8,121 @@ LL |     match_no_arms!(0u8);
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyStruct1` is non-empty
-  --> $DIR/empty-match.rs:121:20
+  --> $DIR/empty-match.rs:47:20
    |
 LL |     match_no_arms!(NonEmptyStruct1);
    |                    ^^^^^^^^^^^^^^^
    |
 note: `NonEmptyStruct1` defined here
-  --> $DIR/empty-match.rs:15:8
+  --> $DIR/empty-match.rs:22:12
    |
-LL | struct NonEmptyStruct1;
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct1;
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct1`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyStruct2` is non-empty
-  --> $DIR/empty-match.rs:123:20
+  --> $DIR/empty-match.rs:48:20
    |
 LL |     match_no_arms!(NonEmptyStruct2(true));
    |                    ^^^^^^^^^^^^^^^^^^^^^
    |
 note: `NonEmptyStruct2` defined here
-  --> $DIR/empty-match.rs:18:8
+  --> $DIR/empty-match.rs:23:12
    |
-LL | struct NonEmptyStruct2(bool);
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct2(bool);
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct2`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyUnion1` is non-empty
-  --> $DIR/empty-match.rs:125:20
+  --> $DIR/empty-match.rs:49:20
    |
 LL |     match_no_arms!((NonEmptyUnion1 { foo: () }));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
 note: `NonEmptyUnion1` defined here
-  --> $DIR/empty-match.rs:21:7
+  --> $DIR/empty-match.rs:24:11
    |
-LL | union NonEmptyUnion1 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion1 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion1`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyUnion2` is non-empty
-  --> $DIR/empty-match.rs:127:20
+  --> $DIR/empty-match.rs:50:20
    |
 LL |     match_no_arms!((NonEmptyUnion2 { foo: () }));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
 note: `NonEmptyUnion2` defined here
-  --> $DIR/empty-match.rs:26:7
+  --> $DIR/empty-match.rs:27:11
    |
-LL | union NonEmptyUnion2 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion2 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion2`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum1::Foo(_)` not covered
-  --> $DIR/empty-match.rs:129:20
+  --> $DIR/empty-match.rs:51:20
    |
 LL |     match_no_arms!(NonEmptyEnum1::Foo(true));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyEnum1::Foo(_)` not covered
    |
 note: `NonEmptyEnum1` defined here
-  --> $DIR/empty-match.rs:32:6
+  --> $DIR/empty-match.rs:31:10
    |
-LL | enum NonEmptyEnum1 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
+LL |     enum NonEmptyEnum1 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum1`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
-  --> $DIR/empty-match.rs:132:20
+  --> $DIR/empty-match.rs:52:20
    |
 LL |     match_no_arms!(NonEmptyEnum2::Foo(true));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
    |
 note: `NonEmptyEnum2` defined here
-  --> $DIR/empty-match.rs:39:6
-   |
-LL | enum NonEmptyEnum2 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
-...
-LL |     Bar,
-   |     --- not covered
+  --> $DIR/empty-match.rs:34:10
+   |
+LL |     enum NonEmptyEnum2 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
+LL |         Bar,
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum2`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or multiple match arms
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
-  --> $DIR/empty-match.rs:135:20
+  --> $DIR/empty-match.rs:53:20
    |
 LL |     match_no_arms!(NonEmptyEnum5::V1);
    |                    ^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
    |
 note: `NonEmptyEnum5` defined here
-  --> $DIR/empty-match.rs:49:6
-   |
-LL | enum NonEmptyEnum5 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     V1, V2, V3, V4, V5,
-   |     --  --  --  --  -- not covered
-   |     |   |   |   |
-   |     |   |   |   not covered
-   |     |   |   not covered
-   |     |   not covered
-   |     not covered
+  --> $DIR/empty-match.rs:38:10
+   |
+LL |     enum NonEmptyEnum5 {
+   |          ^^^^^^^^^^^^^
+LL |         V1,
+   |         -- not covered
+LL |         V2,
+   |         -- not covered
+LL |         V3,
+   |         -- not covered
+LL |         V4,
+   |         -- not covered
+LL |         V5,
+   |         -- not covered
    = note: the matched value is of type `NonEmptyEnum5`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or multiple match arms
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/empty-match.rs:139:24
+  --> $DIR/empty-match.rs:55:24
    |
 LL |     match_guarded_arm!(0u8);
    |                        ^^^ pattern `_` not covered
@@ -189,161 +131,159 @@ LL |     match_guarded_arm!(0u8);
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             _ => todo!()
+LL ~                 _ if false => {},
+LL +                 _ => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyStruct1` not covered
-  --> $DIR/empty-match.rs:144:24
+  --> $DIR/empty-match.rs:56:24
    |
 LL |     match_guarded_arm!(NonEmptyStruct1);
    |                        ^^^^^^^^^^^^^^^ pattern `NonEmptyStruct1` not covered
    |
 note: `NonEmptyStruct1` defined here
-  --> $DIR/empty-match.rs:15:8
+  --> $DIR/empty-match.rs:22:12
    |
-LL | struct NonEmptyStruct1;
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct1;
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct1`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyStruct1 => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyStruct1 => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyStruct2(_)` not covered
-  --> $DIR/empty-match.rs:149:24
+  --> $DIR/empty-match.rs:57:24
    |
 LL |     match_guarded_arm!(NonEmptyStruct2(true));
    |                        ^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyStruct2(_)` not covered
    |
 note: `NonEmptyStruct2` defined here
-  --> $DIR/empty-match.rs:18:8
+  --> $DIR/empty-match.rs:23:12
    |
-LL | struct NonEmptyStruct2(bool);
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct2(bool);
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct2`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyStruct2(_) => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyStruct2(_) => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyUnion1 { .. }` not covered
-  --> $DIR/empty-match.rs:154:24
+  --> $DIR/empty-match.rs:58:24
    |
 LL |     match_guarded_arm!((NonEmptyUnion1 { foo: () }));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyUnion1 { .. }` not covered
    |
 note: `NonEmptyUnion1` defined here
-  --> $DIR/empty-match.rs:21:7
+  --> $DIR/empty-match.rs:24:11
    |
-LL | union NonEmptyUnion1 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion1 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion1`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyUnion1 { .. } => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyUnion1 { .. } => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyUnion2 { .. }` not covered
-  --> $DIR/empty-match.rs:159:24
+  --> $DIR/empty-match.rs:59:24
    |
 LL |     match_guarded_arm!((NonEmptyUnion2 { foo: () }));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyUnion2 { .. }` not covered
    |
 note: `NonEmptyUnion2` defined here
-  --> $DIR/empty-match.rs:26:7
+  --> $DIR/empty-match.rs:27:11
    |
-LL | union NonEmptyUnion2 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion2 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion2`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyUnion2 { .. } => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyUnion2 { .. } => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum1::Foo(_)` not covered
-  --> $DIR/empty-match.rs:164:24
+  --> $DIR/empty-match.rs:60:24
    |
 LL |     match_guarded_arm!(NonEmptyEnum1::Foo(true));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyEnum1::Foo(_)` not covered
    |
 note: `NonEmptyEnum1` defined here
-  --> $DIR/empty-match.rs:32:6
+  --> $DIR/empty-match.rs:31:10
    |
-LL | enum NonEmptyEnum1 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
+LL |     enum NonEmptyEnum1 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum1`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyEnum1::Foo(_) => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyEnum1::Foo(_) => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
-  --> $DIR/empty-match.rs:169:24
+  --> $DIR/empty-match.rs:61:24
    |
 LL |     match_guarded_arm!(NonEmptyEnum2::Foo(true));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
    |
 note: `NonEmptyEnum2` defined here
-  --> $DIR/empty-match.rs:39:6
-   |
-LL | enum NonEmptyEnum2 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
-...
-LL |     Bar,
-   |     --- not covered
+  --> $DIR/empty-match.rs:34:10
+   |
+LL |     enum NonEmptyEnum2 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
+LL |         Bar,
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum2`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
-LL ~             _ if false => {},
-LL +             NonEmptyEnum2::Foo(_) | NonEmptyEnum2::Bar => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyEnum2::Foo(_) | NonEmptyEnum2::Bar => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
-  --> $DIR/empty-match.rs:174:24
+  --> $DIR/empty-match.rs:62:24
    |
 LL |     match_guarded_arm!(NonEmptyEnum5::V1);
    |                        ^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
    |
 note: `NonEmptyEnum5` defined here
-  --> $DIR/empty-match.rs:49:6
-   |
-LL | enum NonEmptyEnum5 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     V1, V2, V3, V4, V5,
-   |     --  --  --  --  -- not covered
-   |     |   |   |   |
-   |     |   |   |   not covered
-   |     |   |   not covered
-   |     |   not covered
-   |     not covered
+  --> $DIR/empty-match.rs:38:10
+   |
+LL |     enum NonEmptyEnum5 {
+   |          ^^^^^^^^^^^^^
+LL |         V1,
+   |         -- not covered
+LL |         V2,
+   |         -- not covered
+LL |         V3,
+   |         -- not covered
+LL |         V4,
+   |         -- not covered
+LL |         V5,
+   |         -- not covered
    = note: the matched value is of type `NonEmptyEnum5`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown, or multiple match arms
    |
-LL ~             _ if false => {},
-LL +             _ => todo!()
+LL ~                 _ if false => {},
+LL +                 _ => todo!()
    |
 
-error: aborting due to 23 previous errors
+error: aborting due to 16 previous errors
 
-Some errors have detailed explanations: E0004, E0005.
-For more information about an error, try `rustc --explain E0004`.
+For more information about this error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-match.normal.stderr b/tests/ui/pattern/usefulness/empty-match.normal.stderr
index 7f0389f40e2..9c3bebd7797 100644
--- a/tests/ui/pattern/usefulness/empty-match.normal.stderr
+++ b/tests/ui/pattern/usefulness/empty-match.normal.stderr
@@ -1,61 +1,5 @@
-error: unreachable pattern
-  --> $DIR/empty-match.rs:68:9
-   |
-LL |         _ => {},
-   |         ^
-   |
-note: the lint level is defined here
-  --> $DIR/empty-match.rs:8:9
-   |
-LL | #![deny(unreachable_patterns)]
-   |         ^^^^^^^^^^^^^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:71:9
-   |
-LL |         _ if false => {},
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:78:9
-   |
-LL |         _ => {},
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:81:9
-   |
-LL |         _ if false => {},
-   |         ^
-
-error[E0005]: refutable pattern in local binding
-  --> $DIR/empty-match.rs:86:9
-   |
-LL |     let None = x;
-   |         ^^^^ pattern `Some(_)` not covered
-   |
-   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
-   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
-   = note: the matched value is of type `Option<SecretlyUninhabitedForeignStruct>`
-help: you might want to use `if let` to ignore the variant that isn't matched
-   |
-LL |     if let None = x { todo!() };
-   |     ++              +++++++++++
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:98:9
-   |
-LL |         _ => {},
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-match.rs:101:9
-   |
-LL |         _ if false => {},
-   |         ^
-
 error[E0004]: non-exhaustive patterns: type `u8` is non-empty
-  --> $DIR/empty-match.rs:119:20
+  --> $DIR/empty-match.rs:46:20
    |
 LL |     match_no_arms!(0u8);
    |                    ^^^
@@ -64,122 +8,121 @@ LL |     match_no_arms!(0u8);
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyStruct1` is non-empty
-  --> $DIR/empty-match.rs:121:20
+  --> $DIR/empty-match.rs:47:20
    |
 LL |     match_no_arms!(NonEmptyStruct1);
    |                    ^^^^^^^^^^^^^^^
    |
 note: `NonEmptyStruct1` defined here
-  --> $DIR/empty-match.rs:15:8
+  --> $DIR/empty-match.rs:22:12
    |
-LL | struct NonEmptyStruct1;
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct1;
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct1`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyStruct2` is non-empty
-  --> $DIR/empty-match.rs:123:20
+  --> $DIR/empty-match.rs:48:20
    |
 LL |     match_no_arms!(NonEmptyStruct2(true));
    |                    ^^^^^^^^^^^^^^^^^^^^^
    |
 note: `NonEmptyStruct2` defined here
-  --> $DIR/empty-match.rs:18:8
+  --> $DIR/empty-match.rs:23:12
    |
-LL | struct NonEmptyStruct2(bool);
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct2(bool);
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct2`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyUnion1` is non-empty
-  --> $DIR/empty-match.rs:125:20
+  --> $DIR/empty-match.rs:49:20
    |
 LL |     match_no_arms!((NonEmptyUnion1 { foo: () }));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
 note: `NonEmptyUnion1` defined here
-  --> $DIR/empty-match.rs:21:7
+  --> $DIR/empty-match.rs:24:11
    |
-LL | union NonEmptyUnion1 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion1 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion1`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: type `NonEmptyUnion2` is non-empty
-  --> $DIR/empty-match.rs:127:20
+  --> $DIR/empty-match.rs:50:20
    |
 LL |     match_no_arms!((NonEmptyUnion2 { foo: () }));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
 note: `NonEmptyUnion2` defined here
-  --> $DIR/empty-match.rs:26:7
+  --> $DIR/empty-match.rs:27:11
    |
-LL | union NonEmptyUnion2 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion2 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion2`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum1::Foo(_)` not covered
-  --> $DIR/empty-match.rs:129:20
+  --> $DIR/empty-match.rs:51:20
    |
 LL |     match_no_arms!(NonEmptyEnum1::Foo(true));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyEnum1::Foo(_)` not covered
    |
 note: `NonEmptyEnum1` defined here
-  --> $DIR/empty-match.rs:32:6
+  --> $DIR/empty-match.rs:31:10
    |
-LL | enum NonEmptyEnum1 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
+LL |     enum NonEmptyEnum1 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum1`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
-  --> $DIR/empty-match.rs:132:20
+  --> $DIR/empty-match.rs:52:20
    |
 LL |     match_no_arms!(NonEmptyEnum2::Foo(true));
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
    |
 note: `NonEmptyEnum2` defined here
-  --> $DIR/empty-match.rs:39:6
-   |
-LL | enum NonEmptyEnum2 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
-...
-LL |     Bar,
-   |     --- not covered
+  --> $DIR/empty-match.rs:34:10
+   |
+LL |     enum NonEmptyEnum2 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
+LL |         Bar,
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum2`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or multiple match arms
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
-  --> $DIR/empty-match.rs:135:20
+  --> $DIR/empty-match.rs:53:20
    |
 LL |     match_no_arms!(NonEmptyEnum5::V1);
    |                    ^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
    |
 note: `NonEmptyEnum5` defined here
-  --> $DIR/empty-match.rs:49:6
-   |
-LL | enum NonEmptyEnum5 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     V1, V2, V3, V4, V5,
-   |     --  --  --  --  -- not covered
-   |     |   |   |   |
-   |     |   |   |   not covered
-   |     |   |   not covered
-   |     |   not covered
-   |     not covered
+  --> $DIR/empty-match.rs:38:10
+   |
+LL |     enum NonEmptyEnum5 {
+   |          ^^^^^^^^^^^^^
+LL |         V1,
+   |         -- not covered
+LL |         V2,
+   |         -- not covered
+LL |         V3,
+   |         -- not covered
+LL |         V4,
+   |         -- not covered
+LL |         V5,
+   |         -- not covered
    = note: the matched value is of type `NonEmptyEnum5`
    = help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or multiple match arms
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/empty-match.rs:139:24
+  --> $DIR/empty-match.rs:55:24
    |
 LL |     match_guarded_arm!(0u8);
    |                        ^^^ pattern `_` not covered
@@ -188,161 +131,159 @@ LL |     match_guarded_arm!(0u8);
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             _ => todo!()
+LL ~                 _ if false => {},
+LL +                 _ => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyStruct1` not covered
-  --> $DIR/empty-match.rs:144:24
+  --> $DIR/empty-match.rs:56:24
    |
 LL |     match_guarded_arm!(NonEmptyStruct1);
    |                        ^^^^^^^^^^^^^^^ pattern `NonEmptyStruct1` not covered
    |
 note: `NonEmptyStruct1` defined here
-  --> $DIR/empty-match.rs:15:8
+  --> $DIR/empty-match.rs:22:12
    |
-LL | struct NonEmptyStruct1;
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct1;
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct1`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyStruct1 => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyStruct1 => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyStruct2(_)` not covered
-  --> $DIR/empty-match.rs:149:24
+  --> $DIR/empty-match.rs:57:24
    |
 LL |     match_guarded_arm!(NonEmptyStruct2(true));
    |                        ^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyStruct2(_)` not covered
    |
 note: `NonEmptyStruct2` defined here
-  --> $DIR/empty-match.rs:18:8
+  --> $DIR/empty-match.rs:23:12
    |
-LL | struct NonEmptyStruct2(bool);
-   |        ^^^^^^^^^^^^^^^
+LL |     struct NonEmptyStruct2(bool);
+   |            ^^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyStruct2`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyStruct2(_) => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyStruct2(_) => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyUnion1 { .. }` not covered
-  --> $DIR/empty-match.rs:154:24
+  --> $DIR/empty-match.rs:58:24
    |
 LL |     match_guarded_arm!((NonEmptyUnion1 { foo: () }));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyUnion1 { .. }` not covered
    |
 note: `NonEmptyUnion1` defined here
-  --> $DIR/empty-match.rs:21:7
+  --> $DIR/empty-match.rs:24:11
    |
-LL | union NonEmptyUnion1 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion1 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion1`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyUnion1 { .. } => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyUnion1 { .. } => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyUnion2 { .. }` not covered
-  --> $DIR/empty-match.rs:159:24
+  --> $DIR/empty-match.rs:59:24
    |
 LL |     match_guarded_arm!((NonEmptyUnion2 { foo: () }));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyUnion2 { .. }` not covered
    |
 note: `NonEmptyUnion2` defined here
-  --> $DIR/empty-match.rs:26:7
+  --> $DIR/empty-match.rs:27:11
    |
-LL | union NonEmptyUnion2 {
-   |       ^^^^^^^^^^^^^^
+LL |     union NonEmptyUnion2 {
+   |           ^^^^^^^^^^^^^^
    = note: the matched value is of type `NonEmptyUnion2`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyUnion2 { .. } => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyUnion2 { .. } => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum1::Foo(_)` not covered
-  --> $DIR/empty-match.rs:164:24
+  --> $DIR/empty-match.rs:60:24
    |
 LL |     match_guarded_arm!(NonEmptyEnum1::Foo(true));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^ pattern `NonEmptyEnum1::Foo(_)` not covered
    |
 note: `NonEmptyEnum1` defined here
-  --> $DIR/empty-match.rs:32:6
+  --> $DIR/empty-match.rs:31:10
    |
-LL | enum NonEmptyEnum1 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
+LL |     enum NonEmptyEnum1 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum1`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL ~             _ if false => {},
-LL +             NonEmptyEnum1::Foo(_) => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyEnum1::Foo(_) => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
-  --> $DIR/empty-match.rs:169:24
+  --> $DIR/empty-match.rs:61:24
    |
 LL |     match_guarded_arm!(NonEmptyEnum2::Foo(true));
    |                        ^^^^^^^^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
    |
 note: `NonEmptyEnum2` defined here
-  --> $DIR/empty-match.rs:39:6
-   |
-LL | enum NonEmptyEnum2 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     Foo(bool),
-   |     --- not covered
-...
-LL |     Bar,
-   |     --- not covered
+  --> $DIR/empty-match.rs:34:10
+   |
+LL |     enum NonEmptyEnum2 {
+   |          ^^^^^^^^^^^^^
+LL |         Foo(bool),
+   |         --- not covered
+LL |         Bar,
+   |         --- not covered
    = note: the matched value is of type `NonEmptyEnum2`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
-LL ~             _ if false => {},
-LL +             NonEmptyEnum2::Foo(_) | NonEmptyEnum2::Bar => todo!()
+LL ~                 _ if false => {},
+LL +                 NonEmptyEnum2::Foo(_) | NonEmptyEnum2::Bar => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
-  --> $DIR/empty-match.rs:174:24
+  --> $DIR/empty-match.rs:62:24
    |
 LL |     match_guarded_arm!(NonEmptyEnum5::V1);
    |                        ^^^^^^^^^^^^^^^^^ patterns `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
    |
 note: `NonEmptyEnum5` defined here
-  --> $DIR/empty-match.rs:49:6
-   |
-LL | enum NonEmptyEnum5 {
-   |      ^^^^^^^^^^^^^
-...
-LL |     V1, V2, V3, V4, V5,
-   |     --  --  --  --  -- not covered
-   |     |   |   |   |
-   |     |   |   |   not covered
-   |     |   |   not covered
-   |     |   not covered
-   |     not covered
+  --> $DIR/empty-match.rs:38:10
+   |
+LL |     enum NonEmptyEnum5 {
+   |          ^^^^^^^^^^^^^
+LL |         V1,
+   |         -- not covered
+LL |         V2,
+   |         -- not covered
+LL |         V3,
+   |         -- not covered
+LL |         V4,
+   |         -- not covered
+LL |         V5,
+   |         -- not covered
    = note: the matched value is of type `NonEmptyEnum5`
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown, or multiple match arms
    |
-LL ~             _ if false => {},
-LL +             _ => todo!()
+LL ~                 _ if false => {},
+LL +                 _ => todo!()
    |
 
-error: aborting due to 23 previous errors
+error: aborting due to 16 previous errors
 
-Some errors have detailed explanations: E0004, E0005.
-For more information about an error, try `rustc --explain E0004`.
+For more information about this error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-match.rs b/tests/ui/pattern/usefulness/empty-match.rs
index fe5d0bce14f..321f24adc46 100644
--- a/tests/ui/pattern/usefulness/empty-match.rs
+++ b/tests/ui/pattern/usefulness/empty-match.rs
@@ -1,179 +1,65 @@
-// aux-build:empty.rs
 // revisions: normal exhaustive_patterns
 //
 // This tests a match with no arms on various types.
 #![feature(never_type)]
-#![feature(never_type_fallback)]
 #![cfg_attr(exhaustive_patterns, feature(exhaustive_patterns))]
 #![deny(unreachable_patterns)]
-//~^ NOTE the lint level is defined here
 
-extern crate empty;
-
-enum EmptyEnum {}
-
-struct NonEmptyStruct1;
-//~^ NOTE `NonEmptyStruct1` defined here
-//~| NOTE `NonEmptyStruct1` defined here
-struct NonEmptyStruct2(bool);
-//~^ NOTE `NonEmptyStruct2` defined here
-//~| NOTE `NonEmptyStruct2` defined here
-union NonEmptyUnion1 {
-    //~^ NOTE `NonEmptyUnion1` defined here
-    //~| NOTE `NonEmptyUnion1` defined here
-    foo: (),
-}
-union NonEmptyUnion2 {
-    //~^ NOTE `NonEmptyUnion2` defined here
-    //~| NOTE `NonEmptyUnion2` defined here
-    foo: (),
-    bar: (),
-}
-enum NonEmptyEnum1 {
-    //~^ NOTE `NonEmptyEnum1` defined here
-    //~| NOTE `NonEmptyEnum1` defined here
-    Foo(bool),
-    //~^ NOTE not covered
-    //~| NOTE not covered
-}
-enum NonEmptyEnum2 {
-    //~^ NOTE `NonEmptyEnum2` defined here
-    //~| NOTE `NonEmptyEnum2` defined here
-    Foo(bool),
-    //~^ NOTE not covered
-    //~| NOTE not covered
-    Bar,
-    //~^ NOTE not covered
-    //~| NOTE not covered
-}
-enum NonEmptyEnum5 {
-    //~^ NOTE `NonEmptyEnum5` defined here
-    //~| NOTE `NonEmptyEnum5` defined here
-    V1, V2, V3, V4, V5,
-    //~^ NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-    //~| NOTE not covered
-}
-
-fn empty_enum(x: EmptyEnum) {
-    match x {} // ok
-    match x {
-        _ => {}, //~ ERROR unreachable pattern
+fn nonempty() {
+    macro_rules! match_no_arms {
+        ($e:expr) => {
+            match $e {}
+        };
     }
-    match x {
-        _ if false => {}, //~ ERROR unreachable pattern
+    macro_rules! match_guarded_arm {
+        ($e:expr) => {
+            match $e {
+                _ if false => {}
+            }
+        };
     }
-}
 
-fn empty_foreign_enum(x: empty::EmptyForeignEnum) {
-    match x {} // ok
-    match x {
-        _ => {}, //~ ERROR unreachable pattern
+    struct NonEmptyStruct1;
+    struct NonEmptyStruct2(bool);
+    union NonEmptyUnion1 {
+        foo: (),
     }
-    match x {
-        _ if false => {}, //~ ERROR unreachable pattern
+    union NonEmptyUnion2 {
+        foo: (),
+        bar: !,
     }
-}
-
-fn empty_foreign_enum_private(x: Option<empty::SecretlyUninhabitedForeignStruct>) {
-    let None = x;
-    //~^ ERROR refutable pattern in local binding
-    //~| NOTE `let` bindings require an "irrefutable pattern"
-    //~| NOTE for more information, visit
-    //~| NOTE the matched value is of type
-    //~| NOTE pattern `Some(_)` not covered
-    //[exhaustive_patterns]~| NOTE currently uninhabited, but this variant contains private fields
-}
-
-fn never(x: !) {
-    match x {} // ok
-    match x {
-        _ => {}, //~ ERROR unreachable pattern
+    enum NonEmptyEnum1 {
+        Foo(bool),
     }
-    match x {
-        _ if false => {}, //~ ERROR unreachable pattern
+    enum NonEmptyEnum2 {
+        Foo(bool),
+        Bar,
+    }
+    enum NonEmptyEnum5 {
+        V1,
+        V2,
+        V3,
+        V4,
+        V5,
     }
-}
-
-macro_rules! match_no_arms {
-    ($e:expr) => {
-        match $e {}
-    };
-}
-macro_rules! match_guarded_arm {
-    ($e:expr) => {
-        match $e {
-            _ if false => {}
-        }
-    };
-}
 
-fn main() {
     match_no_arms!(0u8); //~ ERROR type `u8` is non-empty
-                         //~| NOTE the matched value is of type
     match_no_arms!(NonEmptyStruct1); //~ ERROR type `NonEmptyStruct1` is non-empty
-                                     //~| NOTE the matched value is of type
     match_no_arms!(NonEmptyStruct2(true)); //~ ERROR type `NonEmptyStruct2` is non-empty
-                                           //~| NOTE the matched value is of type
     match_no_arms!((NonEmptyUnion1 { foo: () })); //~ ERROR type `NonEmptyUnion1` is non-empty
-                                                  //~| NOTE the matched value is of type
     match_no_arms!((NonEmptyUnion2 { foo: () })); //~ ERROR type `NonEmptyUnion2` is non-empty
-                                                  //~| NOTE the matched value is of type
     match_no_arms!(NonEmptyEnum1::Foo(true)); //~ ERROR `NonEmptyEnum1::Foo(_)` not covered
-                                              //~| NOTE pattern `NonEmptyEnum1::Foo(_)` not covered
-                                              //~| NOTE the matched value is of type
     match_no_arms!(NonEmptyEnum2::Foo(true)); //~ ERROR `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
-                                              //~| NOTE patterns `NonEmptyEnum2::Foo(_)` and
-                                              //~| NOTE the matched value is of type
     match_no_arms!(NonEmptyEnum5::V1); //~ ERROR `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
-                                       //~| NOTE patterns `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`
-                                       //~| NOTE the matched value is of type
 
     match_guarded_arm!(0u8); //~ ERROR `_` not covered
-                             //~| NOTE the matched value is of type
-                             //~| NOTE match arms with guards don't count towards exhaustivity
-                             //~| NOTE pattern `_` not covered
-                             //~| NOTE in this expansion of match_guarded_arm!
     match_guarded_arm!(NonEmptyStruct1); //~ ERROR `NonEmptyStruct1` not covered
-                                         //~| NOTE pattern `NonEmptyStruct1` not covered
-                                         //~| NOTE the matched value is of type
-                                         //~| NOTE match arms with guards don't count towards exhaustivity
-                                         //~| NOTE in this expansion of match_guarded_arm!
     match_guarded_arm!(NonEmptyStruct2(true)); //~ ERROR `NonEmptyStruct2(_)` not covered
-                                               //~| NOTE the matched value is of type
-                                               //~| NOTE pattern `NonEmptyStruct2(_)` not covered
-                                               //~| NOTE match arms with guards don't count towards exhaustivity
-                                               //~| NOTE in this expansion of match_guarded_arm!
     match_guarded_arm!((NonEmptyUnion1 { foo: () })); //~ ERROR `NonEmptyUnion1 { .. }` not covered
-                                                      //~| NOTE the matched value is of type
-                                                      //~| NOTE pattern `NonEmptyUnion1 { .. }` not covered
-                                                      //~| NOTE match arms with guards don't count towards exhaustivity
-                                                      //~| NOTE in this expansion of match_guarded_arm!
     match_guarded_arm!((NonEmptyUnion2 { foo: () })); //~ ERROR `NonEmptyUnion2 { .. }` not covered
-                                                      //~| NOTE the matched value is of type
-                                                      //~| NOTE pattern `NonEmptyUnion2 { .. }` not covered
-                                                      //~| NOTE match arms with guards don't count towards exhaustivity
-                                                      //~| NOTE in this expansion of match_guarded_arm!
     match_guarded_arm!(NonEmptyEnum1::Foo(true)); //~ ERROR `NonEmptyEnum1::Foo(_)` not covered
-                                                  //~| NOTE the matched value is of type
-                                                  //~| NOTE pattern `NonEmptyEnum1::Foo(_)` not covered
-                                                  //~| NOTE match arms with guards don't count towards exhaustivity
-                                                  //~| NOTE in this expansion of match_guarded_arm!
     match_guarded_arm!(NonEmptyEnum2::Foo(true)); //~ ERROR `NonEmptyEnum2::Foo(_)` and `NonEmptyEnum2::Bar` not covered
-                                                  //~| NOTE the matched value is of type
-                                                  //~| NOTE patterns `NonEmptyEnum2::Foo(_)` and
-                                                  //~| NOTE match arms with guards don't count towards exhaustivity
-                                                  //~| NOTE in this expansion of match_guarded_arm!
     match_guarded_arm!(NonEmptyEnum5::V1); //~ ERROR `NonEmptyEnum5::V1`, `NonEmptyEnum5::V2`, `NonEmptyEnum5::V3` and 2 more not covered
-                                           //~| NOTE the matched value is of type
-                                           //~| NOTE patterns `NonEmptyEnum5::V1`,
-                                           //~| NOTE match arms with guards don't count towards exhaustivity
-                                           //~| NOTE in this expansion of match_guarded_arm!
 }
+
+fn main() {}
diff --git a/tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr b/tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr
new file mode 100644
index 00000000000..9a53b54704e
--- /dev/null
+++ b/tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr
@@ -0,0 +1,776 @@
+error: unreachable pattern
+  --> $DIR/empty-types.rs:47:9
+   |
+LL |         _ => {}
+   |         ^
+   |
+note: the lint level is defined here
+  --> $DIR/empty-types.rs:13:9
+   |
+LL | #![deny(unreachable_patterns)]
+   |         ^^^^^^^^^^^^^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:50:9
+   |
+LL |         _x => {}
+   |         ^^
+
+error[E0004]: non-exhaustive patterns: type `&!` is non-empty
+  --> $DIR/empty-types.rs:54:11
+   |
+LL |     match ref_never {}
+   |           ^^^^^^^^^
+   |
+   = note: the matched value is of type `&!`
+   = note: references are always considered inhabited
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match ref_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:62:9
+   |
+LL |         &_ => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:69:9
+   |
+LL |         (_, _) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:76:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:79:9
+   |
+LL |         (_, _) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:83:9
+   |
+LL |         _ => {}
+   |         ^
+
+error[E0004]: non-exhaustive patterns: `Ok(_)` not covered
+  --> $DIR/empty-types.rs:87:11
+   |
+LL |     match res_u32_never {}
+   |           ^^^^^^^^^^^^^ pattern `Ok(_)` not covered
+   |
+note: `Result<u32, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<u32, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~     match res_u32_never {
+LL +         Ok(_) => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:95:9
+   |
+LL |         Err(_) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:100:9
+   |
+LL |         Err(_) => {}
+   |         ^^^^^^
+
+error[E0004]: non-exhaustive patterns: `Ok(1_u32..=u32::MAX)` not covered
+  --> $DIR/empty-types.rs:97:11
+   |
+LL |     match res_u32_never {
+   |           ^^^^^^^^^^^^^ pattern `Ok(1_u32..=u32::MAX)` not covered
+   |
+note: `Result<u32, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<u32, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         Err(_) => {},
+LL ~         Ok(1_u32..=u32::MAX) => todo!()
+   |
+
+error[E0005]: refutable pattern in local binding
+  --> $DIR/empty-types.rs:104:9
+   |
+LL |     let Ok(_x) = res_u32_never.as_ref();
+   |         ^^^^^^ pattern `Err(_)` not covered
+   |
+   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
+   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
+   = note: the matched value is of type `Result<&u32, &!>`
+help: you might want to use `let else` to handle the variant that isn't matched
+   |
+LL |     let Ok(_x) = res_u32_never.as_ref() else { todo!() };
+   |                                         ++++++++++++++++
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:115:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:119:9
+   |
+LL |         Ok(_) => {}
+   |         ^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:122:9
+   |
+LL |         Ok(_) => {}
+   |         ^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:123:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:126:9
+   |
+LL |         Ok(_) => {}
+   |         ^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:127:9
+   |
+LL |         Err(_) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:136:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:139:13
+   |
+LL |             _ if false => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:148:13
+   |
+LL |             Some(_) => {}
+   |             ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:152:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:158:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:167:13
+   |
+LL |             Some(_) => {}
+   |             ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:171:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:175:13
+   |
+LL |             _a => {}
+   |             ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:180:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:185:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:204:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:209:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:214:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:219:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:225:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:234:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:239:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:245:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:251:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:256:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:262:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:268:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:284:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:287:9
+   |
+LL |         (_, _) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:290:9
+   |
+LL |         Ok(_) => {}
+   |         ^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:291:9
+   |
+LL |         Err(_) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:296:9
+   |
+LL |         &_ => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:299:9
+   |
+LL |         Uninit { value: _ } => {}
+   |         ^^^^^^^^^^^^^^^^^^^
+
+error[E0004]: non-exhaustive patterns: type `&[!]` is non-empty
+  --> $DIR/empty-types.rs:323:11
+   |
+LL |     match slice_never {}
+   |           ^^^^^^^^^^^
+   |
+   = note: the matched value is of type `&[!]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match slice_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:331:9
+   |
+LL |         [_] => {}
+   |         ^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:332:9
+   |
+LL |         [_, _, ..] => {}
+   |         ^^^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:337:9
+   |
+LL |         [_, _, _, ..] => {}
+   |         ^^^^^^^^^^^^^
+
+error[E0004]: non-exhaustive patterns: `&[]` not covered
+  --> $DIR/empty-types.rs:334:11
+   |
+LL |     match slice_never {
+   |           ^^^^^^^^^^^ pattern `&[]` not covered
+   |
+   = note: the matched value is of type `&[!]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         [_, _, _, ..] => {}, &[] => todo!()
+   |                            ++++++++++++++++
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:341:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:345:9
+   |
+LL |         _x => {}
+   |         ^^
+
+error[E0004]: non-exhaustive patterns: `&[]` not covered
+  --> $DIR/empty-types.rs:347:11
+   |
+LL |     match slice_never {
+   |           ^^^^^^^^^^^ pattern `&[]` not covered
+   |
+   = note: the matched value is of type `&[!]`
+   = note: match arms with guards don't count towards exhaustivity
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         &[..] if false => {},
+LL +         &[] => todo!()
+   |
+
+error[E0004]: non-exhaustive patterns: type `[!]` is non-empty
+  --> $DIR/empty-types.rs:353:11
+   |
+LL |     match *slice_never {}
+   |           ^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `[!]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match *slice_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:363:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:366:9
+   |
+LL |         [_, _, _] => {}
+   |         ^^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:369:9
+   |
+LL |         [_, ..] => {}
+   |         ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:375:9
+   |
+LL |         &[_, _, _] => {}
+   |         ^^^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:379:9
+   |
+LL |         &[_x, _, _] => {}
+   |         ^^^^^^^^^^^
+
+error[E0004]: non-exhaustive patterns: type `[!; 0]` is non-empty
+  --> $DIR/empty-types.rs:383:11
+   |
+LL |     match array_0_never {}
+   |           ^^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `[!; 0]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match array_0_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:390:9
+   |
+LL |         _ => {}
+   |         ^
+
+error[E0004]: non-exhaustive patterns: `[]` not covered
+  --> $DIR/empty-types.rs:392:11
+   |
+LL |     match array_0_never {
+   |           ^^^^^^^^^^^^^ pattern `[]` not covered
+   |
+   = note: the matched value is of type `[!; 0]`
+   = note: match arms with guards don't count towards exhaustivity
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         [..] if false => {},
+LL +         [] => todo!()
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:411:9
+   |
+LL |         Some(_) => {}
+   |         ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:416:9
+   |
+LL |         Some(_a) => {}
+   |         ^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:421:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:426:9
+   |
+LL |         _a => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:436:9
+   |
+LL |         &_ => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:444:9
+   |
+LL |         &_a => {}
+   |         ^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:453:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:458:9
+   |
+LL |         _a => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:463:9
+   |
+LL |         &_ => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:468:9
+   |
+LL |         &_a => {}
+   |         ^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:475:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:479:9
+   |
+LL |         _a => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:485:9
+   |
+LL |         ref _a => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:494:9
+   |
+LL |         Some(_) => {}
+   |         ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:499:9
+   |
+LL |         Some(_a) => {}
+   |         ^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:504:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:509:9
+   |
+LL |         _a => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:514:14
+   |
+LL |         _a @ Some(_) => {}
+   |              ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:521:9
+   |
+LL |         ref _a => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:526:18
+   |
+LL |         ref _a @ Some(_) => {}
+   |                  ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:531:18
+   |
+LL |         ref _a @ Some(_b) => {}
+   |                  ^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:538:9
+   |
+LL |         Ok(_) => {}
+   |         ^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:542:9
+   |
+LL |         Ok(_) => {}
+   |         ^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:544:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:549:9
+   |
+LL |         Ok(_a) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:553:9
+   |
+LL |         Ok(_a) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:555:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:559:9
+   |
+LL |         Ok(_a) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:561:9
+   |
+LL |         Err(_) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:569:9
+   |
+LL |         (_, _) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:573:9
+   |
+LL |         (_x, _) => {}
+   |         ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:577:9
+   |
+LL |         (_, _x) => {}
+   |         ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:581:9
+   |
+LL |         (0, _x) => {}
+   |         ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:583:9
+   |
+LL |         (1.., _) => {}
+   |         ^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:598:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:601:9
+   |
+LL |         _x => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:604:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:607:9
+   |
+LL |         _x if false => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:613:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:615:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:622:9
+   |
+LL |         _a if false => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:624:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:629:9
+   |
+LL |         _a if false => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:634:9
+   |
+LL |         &_a if false => {}
+   |         ^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:641:9
+   |
+LL |         Ok(_x) if false => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:643:9
+   |
+LL |         Ok(_) => {}
+   |         ^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:645:9
+   |
+LL |         Err(_) => {}
+   |         ^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:650:9
+   |
+LL |         (_, _x) if false => {}
+   |         ^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:652:9
+   |
+LL |         (_, _) => {}
+   |         ^^^^^^
+
+error: aborting due to 113 previous errors
+
+Some errors have detailed explanations: E0004, E0005.
+For more information about an error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-types.normal.stderr b/tests/ui/pattern/usefulness/empty-types.normal.stderr
new file mode 100644
index 00000000000..b066393a61e
--- /dev/null
+++ b/tests/ui/pattern/usefulness/empty-types.normal.stderr
@@ -0,0 +1,721 @@
+error: unreachable pattern
+  --> $DIR/empty-types.rs:47:9
+   |
+LL |         _ => {}
+   |         ^
+   |
+note: the lint level is defined here
+  --> $DIR/empty-types.rs:13:9
+   |
+LL | #![deny(unreachable_patterns)]
+   |         ^^^^^^^^^^^^^^^^^^^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:50:9
+   |
+LL |         _x => {}
+   |         ^^
+
+error[E0004]: non-exhaustive patterns: type `&!` is non-empty
+  --> $DIR/empty-types.rs:54:11
+   |
+LL |     match ref_never {}
+   |           ^^^^^^^^^
+   |
+   = note: the matched value is of type `&!`
+   = note: references are always considered inhabited
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match ref_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error[E0004]: non-exhaustive patterns: type `(u32, !)` is non-empty
+  --> $DIR/empty-types.rs:66:11
+   |
+LL |     match tuple_half_never {}
+   |           ^^^^^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `(u32, !)`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match tuple_half_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error[E0004]: non-exhaustive patterns: type `(!, !)` is non-empty
+  --> $DIR/empty-types.rs:73:11
+   |
+LL |     match tuple_never {}
+   |           ^^^^^^^^^^^
+   |
+   = note: the matched value is of type `(!, !)`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match tuple_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:83:9
+   |
+LL |         _ => {}
+   |         ^
+
+error[E0004]: non-exhaustive patterns: `Ok(_)` and `Err(_)` not covered
+  --> $DIR/empty-types.rs:87:11
+   |
+LL |     match res_u32_never {}
+   |           ^^^^^^^^^^^^^ patterns `Ok(_)` and `Err(_)` not covered
+   |
+note: `Result<u32, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<u32, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
+   |
+LL ~     match res_u32_never {
+LL +         Ok(_) | Err(_) => todo!(),
+LL +     }
+   |
+
+error[E0004]: non-exhaustive patterns: `Err(_)` not covered
+  --> $DIR/empty-types.rs:89:11
+   |
+LL |     match res_u32_never {
+   |           ^^^^^^^^^^^^^ pattern `Err(_)` not covered
+   |
+note: `Result<u32, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<u32, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         Ok(_) => {},
+LL +         Err(_) => todo!()
+   |
+
+error[E0004]: non-exhaustive patterns: `Ok(1_u32..=u32::MAX)` not covered
+  --> $DIR/empty-types.rs:97:11
+   |
+LL |     match res_u32_never {
+   |           ^^^^^^^^^^^^^ pattern `Ok(1_u32..=u32::MAX)` not covered
+   |
+note: `Result<u32, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<u32, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         Err(_) => {},
+LL ~         Ok(1_u32..=u32::MAX) => todo!()
+   |
+
+error[E0005]: refutable pattern in local binding
+  --> $DIR/empty-types.rs:102:9
+   |
+LL |     let Ok(_x) = res_u32_never;
+   |         ^^^^^^ pattern `Err(_)` not covered
+   |
+   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
+   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
+   = note: the matched value is of type `Result<u32, !>`
+help: you might want to use `let else` to handle the variant that isn't matched
+   |
+LL |     let Ok(_x) = res_u32_never else { todo!() };
+   |                                ++++++++++++++++
+
+error[E0005]: refutable pattern in local binding
+  --> $DIR/empty-types.rs:104:9
+   |
+LL |     let Ok(_x) = res_u32_never.as_ref();
+   |         ^^^^^^ pattern `Err(_)` not covered
+   |
+   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
+   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
+   = note: the matched value is of type `Result<&u32, &!>`
+help: you might want to use `let else` to handle the variant that isn't matched
+   |
+LL |     let Ok(_x) = res_u32_never.as_ref() else { todo!() };
+   |                                         ++++++++++++++++
+
+error[E0005]: refutable pattern in local binding
+  --> $DIR/empty-types.rs:108:9
+   |
+LL |     let Ok(_x) = &res_u32_never;
+   |         ^^^^^^ pattern `&Err(_)` not covered
+   |
+   = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
+   = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
+   = note: the matched value is of type `&Result<u32, !>`
+help: you might want to use `let else` to handle the variant that isn't matched
+   |
+LL |     let Ok(_x) = &res_u32_never else { todo!() };
+   |                                 ++++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `Ok(_)` and `Err(_)` not covered
+  --> $DIR/empty-types.rs:112:11
+   |
+LL |     match result_never {}
+   |           ^^^^^^^^^^^^ patterns `Ok(_)` and `Err(_)` not covered
+   |
+note: `Result<!, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<!, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
+   |
+LL ~     match result_never {
+LL +         Ok(_) | Err(_) => todo!(),
+LL +     }
+   |
+
+error[E0004]: non-exhaustive patterns: `Err(_)` not covered
+  --> $DIR/empty-types.rs:117:11
+   |
+LL |     match result_never {
+   |           ^^^^^^^^^^^^ pattern `Err(_)` not covered
+   |
+note: `Result<!, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<!, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         Ok(_) => {}, Err(_) => todo!()
+   |                    +++++++++++++++++++
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:136:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:139:13
+   |
+LL |             _ if false => {}
+   |             ^
+
+error[E0004]: non-exhaustive patterns: `Some(_)` not covered
+  --> $DIR/empty-types.rs:142:15
+   |
+LL |         match opt_void {
+   |               ^^^^^^^^ pattern `Some(_)` not covered
+   |
+note: `Option<Void>` defined here
+  --> $SRC_DIR/core/src/option.rs:LL:COL
+  ::: $SRC_DIR/core/src/option.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Option<Void>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~             None => {},
+LL +             Some(_) => todo!()
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:158:13
+   |
+LL |             _ => {}
+   |             ^
+
+error[E0004]: non-exhaustive patterns: `Some(_)` not covered
+  --> $DIR/empty-types.rs:161:15
+   |
+LL |         match *ref_opt_void {
+   |               ^^^^^^^^^^^^^ pattern `Some(_)` not covered
+   |
+note: `Option<Void>` defined here
+  --> $SRC_DIR/core/src/option.rs:LL:COL
+  ::: $SRC_DIR/core/src/option.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Option<Void>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~             None => {},
+LL +             Some(_) => todo!()
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:180:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:185:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:204:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:209:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:214:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:219:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:225:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:234:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:239:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:245:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:251:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:256:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:262:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:268:13
+   |
+LL |             _ => {}
+   |             ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:284:9
+   |
+LL |         _ => {}
+   |         ^
+
+error[E0004]: non-exhaustive patterns: type `(u32, !)` is non-empty
+  --> $DIR/empty-types.rs:312:11
+   |
+LL |     match *x {}
+   |           ^^
+   |
+   = note: the matched value is of type `(u32, !)`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match *x {
+LL +         _ => todo!(),
+LL ~     }
+   |
+
+error[E0004]: non-exhaustive patterns: type `(!, !)` is non-empty
+  --> $DIR/empty-types.rs:314:11
+   |
+LL |     match *x {}
+   |           ^^
+   |
+   = note: the matched value is of type `(!, !)`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match *x {
+LL +         _ => todo!(),
+LL ~     }
+   |
+
+error[E0004]: non-exhaustive patterns: `Ok(_)` and `Err(_)` not covered
+  --> $DIR/empty-types.rs:316:11
+   |
+LL |     match *x {}
+   |           ^^ patterns `Ok(_)` and `Err(_)` not covered
+   |
+note: `Result<!, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<!, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
+   |
+LL ~     match *x {
+LL +         Ok(_) | Err(_) => todo!(),
+LL ~     }
+   |
+
+error[E0004]: non-exhaustive patterns: type `[!; 3]` is non-empty
+  --> $DIR/empty-types.rs:318:11
+   |
+LL |     match *x {}
+   |           ^^
+   |
+   = note: the matched value is of type `[!; 3]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match *x {
+LL +         _ => todo!(),
+LL ~     }
+   |
+
+error[E0004]: non-exhaustive patterns: type `&[!]` is non-empty
+  --> $DIR/empty-types.rs:323:11
+   |
+LL |     match slice_never {}
+   |           ^^^^^^^^^^^
+   |
+   = note: the matched value is of type `&[!]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match slice_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error[E0004]: non-exhaustive patterns: `&[_, ..]` not covered
+  --> $DIR/empty-types.rs:325:11
+   |
+LL |     match slice_never {
+   |           ^^^^^^^^^^^ pattern `&[_, ..]` not covered
+   |
+   = note: the matched value is of type `&[!]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         [] => {},
+LL +         &[_, ..] => todo!()
+   |
+
+error[E0004]: non-exhaustive patterns: `&[]`, `&[_]` and `&[_, _]` not covered
+  --> $DIR/empty-types.rs:334:11
+   |
+LL |     match slice_never {
+   |           ^^^^^^^^^^^ patterns `&[]`, `&[_]` and `&[_, _]` not covered
+   |
+   = note: the matched value is of type `&[!]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
+   |
+LL |         [_, _, _, ..] => {}, &[] | &[_] | &[_, _] => todo!()
+   |                            +++++++++++++++++++++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `&[]` and `&[_, ..]` not covered
+  --> $DIR/empty-types.rs:347:11
+   |
+LL |     match slice_never {
+   |           ^^^^^^^^^^^ patterns `&[]` and `&[_, ..]` not covered
+   |
+   = note: the matched value is of type `&[!]`
+   = note: match arms with guards don't count towards exhaustivity
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
+   |
+LL ~         &[..] if false => {},
+LL +         &[] | &[_, ..] => todo!()
+   |
+
+error[E0004]: non-exhaustive patterns: type `[!]` is non-empty
+  --> $DIR/empty-types.rs:353:11
+   |
+LL |     match *slice_never {}
+   |           ^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `[!]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match *slice_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error[E0004]: non-exhaustive patterns: type `[!; 3]` is non-empty
+  --> $DIR/empty-types.rs:360:11
+   |
+LL |     match array_3_never {}
+   |           ^^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `[!; 3]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match array_3_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error[E0004]: non-exhaustive patterns: type `[!; 0]` is non-empty
+  --> $DIR/empty-types.rs:383:11
+   |
+LL |     match array_0_never {}
+   |           ^^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `[!; 0]`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match array_0_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:390:9
+   |
+LL |         _ => {}
+   |         ^
+
+error[E0004]: non-exhaustive patterns: `[]` not covered
+  --> $DIR/empty-types.rs:392:11
+   |
+LL |     match array_0_never {
+   |           ^^^^^^^^^^^^^ pattern `[]` not covered
+   |
+   = note: the matched value is of type `[!; 0]`
+   = note: match arms with guards don't count towards exhaustivity
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         [..] if false => {},
+LL +         [] => todo!()
+   |
+
+error[E0004]: non-exhaustive patterns: `&Some(_)` not covered
+  --> $DIR/empty-types.rs:446:11
+   |
+LL |     match ref_opt_never {
+   |           ^^^^^^^^^^^^^ pattern `&Some(_)` not covered
+   |
+note: `Option<!>` defined here
+  --> $SRC_DIR/core/src/option.rs:LL:COL
+  ::: $SRC_DIR/core/src/option.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `&Option<!>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         &None => {},
+LL +         &Some(_) => todo!()
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:475:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:479:9
+   |
+LL |         _a => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:485:9
+   |
+LL |         ref _a => {}
+   |         ^^^^^^
+
+error[E0004]: non-exhaustive patterns: `Some(_)` not covered
+  --> $DIR/empty-types.rs:487:11
+   |
+LL |     match *ref_opt_never {
+   |           ^^^^^^^^^^^^^^ pattern `Some(_)` not covered
+   |
+note: `Option<!>` defined here
+  --> $SRC_DIR/core/src/option.rs:LL:COL
+  ::: $SRC_DIR/core/src/option.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Option<!>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         None => {},
+LL +         Some(_) => todo!()
+   |
+
+error[E0004]: non-exhaustive patterns: `Err(_)` not covered
+  --> $DIR/empty-types.rs:535:11
+   |
+LL |     match *ref_res_never {
+   |           ^^^^^^^^^^^^^^ pattern `Err(_)` not covered
+   |
+note: `Result<!, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<!, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         Ok(_) => {}, Err(_) => todo!()
+   |                    +++++++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `Err(_)` not covered
+  --> $DIR/empty-types.rs:546:11
+   |
+LL |     match *ref_res_never {
+   |           ^^^^^^^^^^^^^^ pattern `Err(_)` not covered
+   |
+note: `Result<!, !>` defined here
+  --> $SRC_DIR/core/src/result.rs:LL:COL
+  ::: $SRC_DIR/core/src/result.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Result<!, !>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         Ok(_a) => {}, Err(_) => todo!()
+   |                     +++++++++++++++++++
+
+error[E0004]: non-exhaustive patterns: type `(u32, !)` is non-empty
+  --> $DIR/empty-types.rs:565:11
+   |
+LL |     match *ref_tuple_half_never {}
+   |           ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `(u32, !)`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~     match *ref_tuple_half_never {
+LL +         _ => todo!(),
+LL +     }
+   |
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:598:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:601:9
+   |
+LL |         _x => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:604:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:607:9
+   |
+LL |         _x if false => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:613:9
+   |
+LL |         _ if false => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:615:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:622:9
+   |
+LL |         _a if false => {}
+   |         ^^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:624:9
+   |
+LL |         _ => {}
+   |         ^
+
+error: unreachable pattern
+  --> $DIR/empty-types.rs:629:9
+   |
+LL |         _a if false => {}
+   |         ^^
+
+error[E0004]: non-exhaustive patterns: `&_` not covered
+  --> $DIR/empty-types.rs:631:11
+   |
+LL |     match ref_never {
+   |           ^^^^^^^^^ pattern `&_` not covered
+   |
+   = note: the matched value is of type `&!`
+   = note: references are always considered inhabited
+   = note: match arms with guards don't count towards exhaustivity
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         &_a if false => {}, &_ => todo!()
+   |                           +++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `Some(_)` not covered
+  --> $DIR/empty-types.rs:659:11
+   |
+LL |     match *x {
+   |           ^^ pattern `Some(_)` not covered
+   |
+note: `Option<Result<!, !>>` defined here
+  --> $SRC_DIR/core/src/option.rs:LL:COL
+  ::: $SRC_DIR/core/src/option.rs:LL:COL
+   |
+   = note: not covered
+   = note: the matched value is of type `Option<Result<!, !>>`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL ~         None => {},
+LL +         Some(_) => todo!()
+   |
+
+error: aborting due to 66 previous errors
+
+Some errors have detailed explanations: E0004, E0005.
+For more information about an error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-types.rs b/tests/ui/pattern/usefulness/empty-types.rs
new file mode 100644
index 00000000000..fc422298a31
--- /dev/null
+++ b/tests/ui/pattern/usefulness/empty-types.rs
@@ -0,0 +1,665 @@
+// revisions: normal exhaustive_patterns
+//
+// This tests correct handling of empty types in exhaustiveness checking.
+//
+// Most of the subtlety of this file happens in scrutinee places which are not required to hold
+// valid data, namely dereferences and union field accesses. In these cases, empty arms can
+// generally not be omitted, except with `exhaustive_patterns` which ignores this..
+#![feature(never_type)]
+// This feature is useful to avoid `!` falling back to `()` all the time.
+#![feature(never_type_fallback)]
+#![cfg_attr(exhaustive_patterns, feature(exhaustive_patterns))]
+#![allow(dead_code, unreachable_code)]
+#![deny(unreachable_patterns)]
+
+#[derive(Copy, Clone)]
+enum Void {}
+
+/// A bunch of never situations that can't be normally constructed.
+#[derive(Copy, Clone)]
+struct NeverBundle {
+    never: !,
+    void: Void,
+    tuple_never: (!, !),
+    tuple_half_never: (u32, !),
+    array_3_never: [!; 3],
+    result_never: Result<!, !>,
+}
+
+/// A simplified `MaybeUninit` to test union field accesses.
+#[derive(Copy, Clone)]
+union Uninit<T: Copy> {
+    value: T,
+    uninit: (),
+}
+
+impl<T: Copy> Uninit<T> {
+    fn new() -> Self {
+        Self { uninit: () }
+    }
+}
+
+// Simple cases of omitting empty arms, all with known_valid scrutinees.
+fn basic(x: NeverBundle) {
+    let never: ! = x.never;
+    match never {}
+    match never {
+        _ => {} //~ ERROR unreachable pattern
+    }
+    match never {
+        _x => {} //~ ERROR unreachable pattern
+    }
+
+    let ref_never: &! = &x.never;
+    match ref_never {}
+    //~^ ERROR non-empty
+    match ref_never {
+        // useful, reachable
+        _ => {}
+    }
+    match ref_never {
+        // useful, reachable
+        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    let tuple_half_never: (u32, !) = x.tuple_half_never;
+    match tuple_half_never {}
+    //[normal]~^ ERROR non-empty
+    match tuple_half_never {
+        (_, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    let tuple_never: (!, !) = x.tuple_never;
+    match tuple_never {}
+    //[normal]~^ ERROR non-empty
+    match tuple_never {
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match tuple_never {
+        (_, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match tuple_never.0 {}
+    match tuple_never.0 {
+        _ => {} //~ ERROR unreachable pattern
+    }
+
+    let res_u32_never: Result<u32, !> = Ok(0);
+    match res_u32_never {}
+    //~^ ERROR non-exhaustive
+    match res_u32_never {
+        //[normal]~^ ERROR non-exhaustive
+        Ok(_) => {}
+    }
+    match res_u32_never {
+        Ok(_) => {}
+        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match res_u32_never {
+        //~^ ERROR non-exhaustive
+        Ok(0) => {}
+        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    let Ok(_x) = res_u32_never;
+    //[normal]~^ ERROR refutable
+    let Ok(_x) = res_u32_never.as_ref();
+    //~^ ERROR refutable
+    // Non-obvious difference: here there's an implicit dereference in the patterns, which makes the
+    // inner place !known_valid. `exhaustive_patterns` ignores this.
+    let Ok(_x) = &res_u32_never;
+    //[normal]~^ ERROR refutable
+
+    let result_never: Result<!, !> = x.result_never;
+    match result_never {}
+    //[normal]~^ ERROR non-exhaustive
+    match result_never {
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match result_never {
+        //[normal]~^ ERROR non-exhaustive
+        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match result_never {
+        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _ => {}     //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match result_never {
+        Ok(_) => {}  //[exhaustive_patterns]~ ERROR unreachable pattern
+        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+}
+
+// Check for a few cases that `Void` and `!` are treated the same.
+fn void_same_as_never(x: NeverBundle) {
+    unsafe {
+        match x.void {}
+        match x.void {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        match x.void {
+            _ if false => {} //~ ERROR unreachable pattern
+        }
+        let opt_void: Option<Void> = None;
+        match opt_void {
+            //[normal]~^ ERROR non-exhaustive
+            None => {}
+        }
+        match opt_void {
+            None => {}
+            Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        }
+        match opt_void {
+            None => {}
+            _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        }
+
+        let ref_void: &Void = &x.void;
+        match *ref_void {}
+        match *ref_void {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        let ref_opt_void: &Option<Void> = &None;
+        match *ref_opt_void {
+            //[normal]~^ ERROR non-exhaustive
+            None => {}
+        }
+        match *ref_opt_void {
+            None => {}
+            Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        }
+        match *ref_opt_void {
+            None => {}
+            _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        }
+        match *ref_opt_void {
+            None => {}
+            _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        }
+        let union_void = Uninit::<Void>::new();
+        match union_void.value {}
+        match union_void.value {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        let ptr_void: *const Void = std::ptr::null();
+        match *ptr_void {}
+        match *ptr_void {
+            _ => {} //~ ERROR unreachable pattern
+        }
+    }
+}
+
+// Test if we correctly determine validity from the scrutinee expression.
+fn invalid_scrutinees(x: NeverBundle) {
+    let ptr_never: *const ! = std::ptr::null();
+    let never: ! = x.never;
+    let ref_never: &! = &never;
+
+    struct NestedNeverBundle(NeverBundle);
+    let nested_x = NestedNeverBundle(x);
+
+    // These should be considered known_valid and warn unreachable.
+    unsafe {
+        // A plain `!` value must be valid.
+        match never {}
+        match never {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // A block forces a copy.
+        match { *ptr_never } {}
+        match { *ptr_never } {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // This field access is not a dereference.
+        match x.never {}
+        match x.never {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // This nested field access is not a dereference.
+        match nested_x.0.never {}
+        match nested_x.0.never {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // Indexing is like a field access. This one does not access behind a reference.
+        let array_3_never: [!; 3] = x.array_3_never;
+        match array_3_never[0] {}
+        match array_3_never[0] {
+            _ => {} //~ ERROR unreachable pattern
+        }
+    }
+
+    // These should be considered !known_valid and not warn unreachable.
+    unsafe {
+        // A pointer may point to a place with an invalid value.
+        match *ptr_never {}
+        match *ptr_never {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // A reference may point to a place with an invalid value.
+        match *ref_never {}
+        match *ref_never {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // This field access is a dereference.
+        let ref_x: &NeverBundle = &x;
+        match ref_x.never {}
+        match ref_x.never {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // This nested field access is a dereference.
+        let nested_ref_x: &NestedNeverBundle = &nested_x;
+        match nested_ref_x.0.never {}
+        match nested_ref_x.0.never {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // A cast does not load.
+        match (*ptr_never as Void) {}
+        match (*ptr_never as Void) {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // A union field may contain invalid data.
+        let union_never = Uninit::<!>::new();
+        match union_never.value {}
+        match union_never.value {
+            _ => {} //~ ERROR unreachable pattern
+        }
+        // Indexing is like a field access. This one accesses behind a reference.
+        let slice_never: &[!] = &[];
+        match slice_never[0] {}
+        match slice_never[0] {
+            _ => {} //~ ERROR unreachable pattern
+        }
+    }
+}
+
+// Test we correctly track validity as we dig into patterns. Validity changes when we go under a
+// dereference or a union field access, and it otherwise preserved.
+fn nested_validity_tracking(bundle: NeverBundle) {
+    let never: ! = bundle.never;
+    let ref_never: &! = &never;
+    let tuple_never: (!, !) = bundle.tuple_never;
+    let result_never: Result<!, !> = bundle.result_never;
+    let union_never = Uninit::<!>::new();
+
+    // These should be considered known_valid and warn unreachable.
+    match never {
+        _ => {} //~ ERROR unreachable pattern
+    }
+    match tuple_never {
+        (_, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match result_never {
+        Ok(_) => {}  //[exhaustive_patterns]~ ERROR unreachable pattern
+        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    // These should be considered !known_valid and not warn unreachable.
+    match ref_never {
+        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match union_never {
+        Uninit { value: _ } => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+}
+
+// Test we don't allow empty matches on empty types if the scrutinee is `!known_valid`.
+fn invalid_empty_match(bundle: NeverBundle) {
+    // We allow these two for backwards-compability.
+    let x: &! = &bundle.never;
+    match *x {}
+    let x: &Void = &bundle.void;
+    match *x {}
+
+    let x: &(u32, !) = &bundle.tuple_half_never;
+    match *x {} //[normal]~ ERROR non-exhaustive
+    let x: &(!, !) = &bundle.tuple_never;
+    match *x {} //[normal]~ ERROR non-exhaustive
+    let x: &Result<!, !> = &bundle.result_never;
+    match *x {} //[normal]~ ERROR non-exhaustive
+    let x: &[!; 3] = &bundle.array_3_never;
+    match *x {} //[normal]~ ERROR non-exhaustive
+}
+
+fn arrays_and_slices(x: NeverBundle) {
+    let slice_never: &[!] = &[];
+    match slice_never {}
+    //~^ ERROR non-empty
+    match slice_never {
+        //[normal]~^ ERROR not covered
+        [] => {}
+    }
+    match slice_never {
+        [] => {}
+        [_] => {}        //[exhaustive_patterns]~ ERROR unreachable pattern
+        [_, _, ..] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match slice_never {
+        //[normal]~^ ERROR `&[]`, `&[_]` and `&[_, _]` not covered
+        //[exhaustive_patterns]~^^ ERROR `&[]` not covered
+        [_, _, _, ..] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match slice_never {
+        [] => {}
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match slice_never {
+        [] => {}
+        _x => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match slice_never {
+        //[normal]~^ ERROR `&[]` and `&[_, ..]` not covered
+        //[exhaustive_patterns]~^^ ERROR `&[]` not covered
+        &[..] if false => {}
+    }
+
+    match *slice_never {}
+    //~^ ERROR non-empty
+    match *slice_never {
+        _ => {}
+    }
+
+    let array_3_never: [!; 3] = x.array_3_never;
+    match array_3_never {}
+    //[normal]~^ ERROR non-empty
+    match array_3_never {
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match array_3_never {
+        [_, _, _] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match array_3_never {
+        [_, ..] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    let ref_array_3_never: &[!; 3] = &array_3_never;
+    match ref_array_3_never {
+        // useful, reachable
+        &[_, _, _] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match ref_array_3_never {
+        // useful, !reachable
+        &[_x, _, _] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    let array_0_never: [!; 0] = [];
+    match array_0_never {}
+    //~^ ERROR non-empty
+    match array_0_never {
+        [] => {}
+    }
+    match array_0_never {
+        [] => {}
+        _ => {} //~ ERROR unreachable pattern
+    }
+    match array_0_never {
+        //~^ ERROR `[]` not covered
+        [..] if false => {}
+    }
+}
+
+// The difference between `_` and `_a` patterns is that `_a` loads the value. In case of an empty
+// type, this asserts validity of the value, and thus the binding is unreachable. We don't yet
+// distinguish these cases since we don't lint "unreachable" on `useful && !reachable` arms.
+// Once/if never patterns are a thing, we can warn that the `_a` cases should be never patterns.
+fn bindings(x: NeverBundle) {
+    let opt_never: Option<!> = None;
+    let ref_never: &! = &x.never;
+    let ref_opt_never: &Option<!> = &None;
+
+    // On a known_valid place.
+    match opt_never {
+        None => {}
+        // !useful, !reachable
+        Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match opt_never {
+        None => {}
+        // !useful, !reachable
+        Some(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match opt_never {
+        None => {}
+        // !useful, !reachable
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match opt_never {
+        None => {}
+        // !useful, !reachable
+        _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    // The scrutinee is known_valid, but under the `&` isn't anymore.
+    match ref_never {
+        // useful, reachable
+        _ => {}
+    }
+    match ref_never {
+        // useful, reachable
+        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match ref_never {
+        // useful, reachable
+        _a => {}
+    }
+    match ref_never {
+        // useful, !reachable
+        &_a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match ref_opt_never {
+        //[normal]~^ ERROR non-exhaustive
+        &None => {}
+    }
+    match ref_opt_never {
+        &None => {}
+        // useful, reachable
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match ref_opt_never {
+        &None => {}
+        // useful, reachable
+        _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match ref_opt_never {
+        &None => {}
+        // useful, reachable
+        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match ref_opt_never {
+        &None => {}
+        // useful, !reachable
+        &_a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    // On a !known_valid place.
+    match *ref_never {}
+    match *ref_never {
+        // useful, reachable
+        _ => {} //~ ERROR unreachable pattern
+    }
+    match *ref_never {
+        // useful, !reachable
+        _a => {} //~ ERROR unreachable pattern
+    }
+    // This is equivalent to `match ref_never { _a => {} }`. In other words, it asserts validity of
+    // `ref_never` but says nothing of the data at `*ref_never`.
+    match *ref_never {
+        // useful, reachable
+        ref _a => {} //~ ERROR unreachable pattern
+    }
+    match *ref_opt_never {
+        //[normal]~^ ERROR non-exhaustive
+        None => {}
+    }
+    match *ref_opt_never {
+        None => {}
+        // useful, reachable
+        Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_opt_never {
+        None => {}
+        // useful, !reachable
+        Some(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_opt_never {
+        None => {}
+        // useful, reachable
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_opt_never {
+        None => {}
+        // useful, !reachable
+        _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_opt_never {
+        None => {}
+        // useful, !reachable
+        _a @ Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    // This is equivalent to `match ref_opt_never { None => {}, _a => {} }`. In other words, it
+    // asserts validity of `ref_opt_never` but says nothing of the data at `*ref_opt_never`.
+    match *ref_opt_never {
+        None => {}
+        // useful, reachable
+        ref _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_opt_never {
+        None => {}
+        // useful, reachable
+        ref _a @ Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_opt_never {
+        None => {}
+        // useful, !reachable
+        ref _a @ Some(_b) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    let ref_res_never: &Result<!, !> = &x.result_never;
+    match *ref_res_never {
+        //[normal]~^ ERROR non-exhaustive
+        // useful, reachable
+        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_res_never {
+        // useful, reachable
+        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        // useful, reachable
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_res_never {
+        //[normal]~^ ERROR non-exhaustive
+        // useful, !reachable
+        Ok(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_res_never {
+        // useful, !reachable
+        Ok(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        // useful, reachable
+        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_res_never {
+        // useful, !reachable
+        Ok(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        // useful, reachable
+        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    let ref_tuple_half_never: &(u32, !) = &x.tuple_half_never;
+    match *ref_tuple_half_never {}
+    //[normal]~^ ERROR non-empty
+    match *ref_tuple_half_never {
+        // useful, reachable
+        (_, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_tuple_half_never {
+        // useful, reachable
+        (_x, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_tuple_half_never {
+        // useful, !reachable
+        (_, _x) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    match *ref_tuple_half_never {
+        // useful, !reachable
+        (0, _x) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        // useful, reachable
+        (1.., _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+}
+
+// When we execute the condition for a guard we loads from all bindings. This asserts validity at
+// all places with bindings. Surprisingly this can make subsequent arms unreachable. We choose to
+// not detect this in exhaustiveness because this is rather subtle. With never patterns, we would
+// recommend using a never pattern instead.
+fn guards_and_validity(x: NeverBundle) {
+    let never: ! = x.never;
+    let ref_never: &! = &never;
+
+    // Basic guard behavior when known_valid.
+    match never {}
+    match never {
+        _ => {} //~ ERROR unreachable pattern
+    }
+    match never {
+        _x => {} //~ ERROR unreachable pattern
+    }
+    match never {
+        _ if false => {} //~ ERROR unreachable pattern
+    }
+    match never {
+        _x if false => {} //~ ERROR unreachable pattern
+    }
+
+    // If the pattern under the guard doesn't load, all is normal.
+    match *ref_never {
+        // useful, reachable
+        _ if false => {} //~ ERROR unreachable pattern
+        // useful, reachable
+        _ => {} //~ ERROR unreachable pattern
+    }
+    // Now the madness commences. The guard caused a load of the value thus asserting validity. So
+    // there's no invalid value for `_` to catch. So the second pattern is unreachable despite the
+    // guard not being taken.
+    match *ref_never {
+        // useful, !reachable
+        _a if false => {} //~ ERROR unreachable pattern
+        // !useful, !reachable
+        _ => {} //~ ERROR unreachable pattern
+    }
+    // The above still applies to the implicit `_` pattern used for exhaustiveness.
+    match *ref_never {
+        // useful, !reachable
+        _a if false => {} //~ ERROR unreachable pattern
+    }
+    match ref_never {
+        //[normal]~^ ERROR non-exhaustive
+        // useful, !reachable
+        &_a if false => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+
+    // Same but with subpatterns.
+    let ref_result_never: &Result<!, !> = &x.result_never;
+    match *ref_result_never {
+        // useful, !reachable
+        Ok(_x) if false => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        // !useful, !reachable
+        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        // useful, !reachable
+        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+    let ref_tuple_never: &(!, !) = &x.tuple_never;
+    match *ref_tuple_never {
+        // useful, !reachable
+        (_, _x) if false => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        // !useful, !reachable
+        (_, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+    }
+}
+
+fn diagnostics_subtlety(x: NeverBundle) {
+    // Regression test for diagnostics: don't report `Some(Ok(_))` and `Some(Err(_))`.
+    let x: &Option<Result<!, !>> = &None;
+    match *x {
+        //[normal]~^ ERROR `Some(_)` not covered
+        None => {}
+    }
+}
+
+fn main() {}
diff --git a/tests/ui/uninhabited/uninhabited-patterns.rs b/tests/ui/uninhabited/uninhabited-patterns.rs
index f1573b6adf0..43b19e790e2 100644
--- a/tests/ui/uninhabited/uninhabited-patterns.rs
+++ b/tests/ui/uninhabited/uninhabited-patterns.rs
@@ -1,8 +1,6 @@
 #![feature(box_patterns)]
 #![feature(never_type)]
 #![feature(exhaustive_patterns)]
-
-
 #![deny(unreachable_patterns)]
 
 mod foo {
@@ -23,22 +21,22 @@ fn main() {
     let x: &[!] = &[];
 
     match x {
-        &[]   => (),
-        &[..] => (),    //~ ERROR unreachable pattern
+        &[] => (),
+        &[..] => (), //~ ERROR unreachable pattern
     };
 
     let x: Result<Box<NotSoSecretlyEmpty>, &[Result<!, !>]> = Err(&[]);
     match x {
-        Ok(box _) => (),    //~ ERROR unreachable pattern
+        Ok(box _) => (), //~ ERROR unreachable pattern
         Err(&[]) => (),
-        Err(&[..]) => (),   //~ ERROR unreachable pattern
+        Err(&[..]) => (), //~ ERROR unreachable pattern
     }
 
     let x: Result<foo::SecretlyEmpty, Result<NotSoSecretlyEmpty, u32>> = Err(Err(123));
     match x {
         Ok(_y) => (),
         Err(Err(_y)) => (),
-        Err(Ok(_y)) => (),  //~ ERROR unreachable pattern
+        Err(Ok(_y)) => (), //~ ERROR unreachable pattern
     }
 
     while let Some(_y) = foo() {
diff --git a/tests/ui/uninhabited/uninhabited-patterns.stderr b/tests/ui/uninhabited/uninhabited-patterns.stderr
index 655569ad6e0..19f34a52bdb 100644
--- a/tests/ui/uninhabited/uninhabited-patterns.stderr
+++ b/tests/ui/uninhabited/uninhabited-patterns.stderr
@@ -1,35 +1,35 @@
 error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:27:9
+  --> $DIR/uninhabited-patterns.rs:25:9
    |
 LL |         &[..] => (),
    |         ^^^^^
    |
 note: the lint level is defined here
-  --> $DIR/uninhabited-patterns.rs:6:9
+  --> $DIR/uninhabited-patterns.rs:4:9
    |
 LL | #![deny(unreachable_patterns)]
    |         ^^^^^^^^^^^^^^^^^^^^
 
 error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:32:9
+  --> $DIR/uninhabited-patterns.rs:30:9
    |
 LL |         Ok(box _) => (),
    |         ^^^^^^^^^
 
 error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:34:9
+  --> $DIR/uninhabited-patterns.rs:32:9
    |
 LL |         Err(&[..]) => (),
    |         ^^^^^^^^^^
 
 error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:41:9
+  --> $DIR/uninhabited-patterns.rs:39:9
    |
 LL |         Err(Ok(_y)) => (),
    |         ^^^^^^^^^^^
 
 error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:44:15
+  --> $DIR/uninhabited-patterns.rs:42:15
    |
 LL |     while let Some(_y) = foo() {
    |               ^^^^^^^^
-- 
cgit 1.4.1-3-g733a5


From 9aafc0b815a86924996c31943f6e2de0a0a6f973 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 18 Nov 2023 04:17:50 +0100
Subject: Be precise about usefulness vs reachability

---
 .../src/thir/pattern/check_match.rs                |  18 +-
 .../src/thir/pattern/deconstruct_pat.rs            |  37 ++--
 .../rustc_mir_build/src/thir/pattern/usefulness.rs | 198 ++++++++++++++-------
 3 files changed, 159 insertions(+), 94 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index b72b9da21b7..89b7b3619be 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -1,6 +1,6 @@
 use super::deconstruct_pat::{Constructor, DeconstructedPat, WitnessPat};
 use super::usefulness::{
-    compute_match_usefulness, MatchArm, MatchCheckCtxt, Reachability, UsefulnessReport,
+    compute_match_usefulness, MatchArm, MatchCheckCtxt, Usefulness, UsefulnessReport,
 };
 
 use crate::errors::*;
@@ -749,18 +749,18 @@ fn report_arm_reachability<'p, 'tcx>(
         );
     };
 
-    use Reachability::*;
+    use Usefulness::*;
     let mut catchall = None;
     for (arm, is_useful) in report.arm_usefulness.iter() {
         match is_useful {
-            Unreachable => report_unreachable_pattern(arm.pat.span(), arm.hir_id, catchall),
-            Reachable(unreachables) if unreachables.is_empty() => {}
-            // The arm is reachable, but contains unreachable subpatterns (from or-patterns).
-            Reachable(unreachables) => {
-                let mut unreachables = unreachables.clone();
+            Redundant => report_unreachable_pattern(arm.pat.span(), arm.hir_id, catchall),
+            Useful(redundant_spans) if redundant_spans.is_empty() => {}
+            // The arm is reachable, but contains redundant subpatterns (from or-patterns).
+            Useful(redundant_spans) => {
+                let mut redundant_spans = redundant_spans.clone();
                 // Emit lints in the order in which they occur in the file.
-                unreachables.sort_unstable();
-                for span in unreachables {
+                redundant_spans.sort_unstable();
+                for span in redundant_spans {
                     report_unreachable_pattern(span, arm.hir_id, None);
                 }
             }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index ddcceeb7ef6..094acbc3f8a 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -1339,7 +1339,8 @@ pub(crate) struct DeconstructedPat<'p, 'tcx> {
     fields: Fields<'p, 'tcx>,
     ty: Ty<'tcx>,
     span: Span,
-    reachable: Cell<bool>,
+    /// Whether removing this arm would change the behavior of the match expression.
+    useful: Cell<bool>,
 }
 
 impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
@@ -1353,7 +1354,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
         ty: Ty<'tcx>,
         span: Span,
     ) -> Self {
-        DeconstructedPat { ctor, fields, ty, span, reachable: Cell::new(false) }
+        DeconstructedPat { ctor, fields, ty, span, useful: Cell::new(false) }
     }
 
     /// Note: the input patterns must have been lowered through
@@ -1634,38 +1635,38 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
         }
     }
 
-    /// We keep track for each pattern if it was ever reachable during the analysis. This is used
-    /// with `unreachable_spans` to report unreachable subpatterns arising from or patterns.
-    pub(super) fn set_reachable(&self) {
-        self.reachable.set(true)
+    /// We keep track for each pattern if it was ever useful during the analysis. This is used
+    /// with `redundant_spans` to report redundant subpatterns arising from or patterns.
+    pub(super) fn set_useful(&self) {
+        self.useful.set(true)
     }
-    pub(super) fn is_reachable(&self) -> bool {
-        if self.reachable.get() {
+    pub(super) fn is_useful(&self) -> bool {
+        if self.useful.get() {
             true
-        } else if self.is_or_pat() && self.iter_fields().any(|f| f.is_reachable()) {
+        } else if self.is_or_pat() && self.iter_fields().any(|f| f.is_useful()) {
             // We always expand or patterns in the matrix, so we will never see the actual
             // or-pattern (the one with constructor `Or`) in the column. As such, it will not be
-            // marked as reachable itself, only its children will. We recover this information here.
-            self.set_reachable();
+            // marked as useful itself, only its children will. We recover this information here.
+            self.set_useful();
             true
         } else {
             false
         }
     }
 
-    /// Report the spans of subpatterns that were not reachable, if any.
-    pub(super) fn unreachable_spans(&self) -> Vec<Span> {
+    /// Report the spans of subpatterns that were not useful, if any.
+    pub(super) fn redundant_spans(&self) -> Vec<Span> {
         let mut spans = Vec::new();
-        self.collect_unreachable_spans(&mut spans);
+        self.collect_redundant_spans(&mut spans);
         spans
     }
-    fn collect_unreachable_spans(&self, spans: &mut Vec<Span>) {
-        // We don't look at subpatterns if we already reported the whole pattern as unreachable.
-        if !self.is_reachable() {
+    fn collect_redundant_spans(&self, spans: &mut Vec<Span>) {
+        // We don't look at subpatterns if we already reported the whole pattern as redundant.
+        if !self.is_useful() {
             spans.push(self.span);
         } else {
             for p in self.iter_fields() {
-                p.collect_unreachable_spans(spans);
+                p.collect_redundant_spans(spans);
             }
         }
     }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index a2f829f93e3..e77f158a980 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -1,8 +1,8 @@
-//! # Match exhaustiveness and reachability algorithm
+//! # Match exhaustiveness and redundancy algorithm
 //!
-//! This file contains the logic for exhaustiveness and reachability checking for pattern-matching.
+//! This file contains the logic for exhaustiveness and usefulness checking for pattern-matching.
 //! Specifically, given a list of patterns in a match, we can tell whether:
-//! (a) a given pattern is reachable (reachability)
+//! (a) a given pattern is redundant
 //! (b) the patterns cover every possible value for the type (exhaustiveness)
 //!
 //! The algorithm implemented here is inspired from the one described in [this
@@ -19,15 +19,15 @@
 //! The algorithm is given as input a list of patterns, one for each arm of a match, and computes
 //! the following:
 //! - a set of values that match none of the patterns (if any),
-//! - for each subpattern (taking into account or-patterns), whether it would catch any value that
-//!     isn't caught by a pattern before it, i.e. whether it is reachable.
+//! - for each subpattern (taking into account or-patterns), whether removing it would change
+//!     anything about how the match executes, i.e. whether it is useful/not redundant.
 //!
 //! To a first approximation, the algorithm works by exploring all possible values for the type
 //! being matched on, and determining which arm(s) catch which value. To make this tractable we
 //! cleverly group together values, as we'll see below.
 //!
 //! The entrypoint of this file is the [`compute_match_usefulness`] function, which computes
-//! reachability for each subpattern and exhaustiveness for the whole match.
+//! usefulness for each subpattern and exhaustiveness for the whole match.
 //!
 //! In this page we explain the necessary concepts to understand how the algorithm works.
 //!
@@ -39,17 +39,17 @@
 //! none of the `p_i`. We write `usefulness(p_1 .. p_n, q)` for a function that returns a list of
 //! such values. The aim of this file is to compute it efficiently.
 //!
-//! This is enough to compute reachability: a pattern in a `match` expression is reachable iff it is
-//! useful w.r.t. the patterns above it:
+//! This is enough to compute usefulness: a pattern in a `match` expression is redundant iff it is
+//! not useful w.r.t. the patterns above it:
 //! ```compile_fail,E0004
 //! # #![feature(exclusive_range_pattern)]
 //! # fn foo() {
 //! match Some(0u32) {
 //!     Some(0..100) => {},
-//!     Some(90..190) => {}, // reachable: `Some(150)` is matched by this but not the branch above
-//!     Some(50..150) => {}, // unreachable: all the values this matches are already matched by
+//!     Some(90..190) => {}, // useful: `Some(150)` is matched by this but not the branch above
+//!     Some(50..150) => {}, // redundant: all the values this matches are already matched by
 //!                          //   the branches above
-//!     None => {},          // reachable: `None` is matched by this but not the branches above
+//!     None => {},          // useful: `None` is matched by this but not the branches above
 //! }
 //! # }
 //! ```
@@ -246,18 +246,17 @@
 //!
 //!
 //!
-//! # Computing reachability and exhaustiveness in one go
+//! # Computing usefulness and exhaustiveness in one go
 //!
-//! The algorithm we have described so far computes usefulness of each pattern in turn to check if
-//! it is reachable, and ends by checking if `_` is useful to determine exhaustiveness of the whole
-//! match. In practice, instead of doing "for each pattern { for each constructor { ... } }", we do
-//! "for each constructor { for each pattern { ... } }". This allows us to compute everything in one
-//! go.
+//! The algorithm we have described so far computes usefulness of each pattern in turn, and ends by
+//! checking if `_` is useful to determine exhaustiveness of the whole match. In practice, instead
+//! of doing "for each pattern { for each constructor { ... } }", we do "for each constructor { for
+//! each pattern { ... } }". This allows us to compute everything in one go.
 //!
-//! [`Matrix`] stores the set of pattern-tuples under consideration. We track reachability of each
+//! [`Matrix`] stores the set of pattern-tuples under consideration. We track usefulness of each
 //! row mutably in the matrix as we go along. We ignore witnesses of usefulness of the match rows.
 //! We gather witnesses of the usefulness of `_` in [`WitnessMatrix`]. The algorithm that computes
-//! all this is in [`compute_exhaustiveness_and_reachability`].
+//! all this is in [`compute_exhaustiveness_and_usefulness`].
 //!
 //! See the full example at the bottom of this documentation.
 //!
@@ -279,7 +278,7 @@
 //! ```
 //!
 //! In this example, trying any of `0`, `1`, .., `49` will give the same specialized matrix, and
-//! thus the same reachability/exhaustiveness results. We can thus accelerate the algorithm by
+//! thus the same usefulness/exhaustiveness results. We can thus accelerate the algorithm by
 //! trying them all at once. Here in fact, the only cases we need to consider are: `0..50`,
 //! `50..=100`, `101..=150`,`151..=200` and `201..`.
 //!
@@ -299,15 +298,16 @@
 //! This is done in [`ConstructorSet::split`] and explained in [`super::deconstruct_pat`].
 //!
 //!
+//!
 //! # Or-patterns
 //!
 //! What we have described so far works well if there are no or-patterns. To handle them, if the
 //! first pattern of a row in the matrix is an or-pattern, we expand it by duplicating the rest of
 //! the row as necessary. This is handled automatically in [`Matrix`].
 //!
-//! This makes reachability tracking subtle, because we also want to compute whether an alternative
-//! of an or-pattern is unreachable, e.g. in `Some(_) | Some(0)`. We track reachability of each
-//! subpattern by interior mutability in [`DeconstructedPat`] with `set_reachable`/`is_reachable`.
+//! This makes usefulness tracking subtle, because we also want to compute whether an alternative
+//! of an or-pattern is redundant, e.g. in `Some(_) | Some(0)`. We track usefulness of each
+//! subpattern by interior mutability in [`DeconstructedPat`] with `set_useful`/`is_useful`.
 //!
 //! It's unfortunate that we have to use interior mutability, but believe me (Nadrieril), I have
 //! tried [other](https://github.com/rust-lang/rust/pull/80104)
@@ -332,6 +332,69 @@
 //!
 //!
 //!
+//! # Usefulness vs reachability, validity, and empty patterns
+//!
+//! This is likely the subtlest aspect of the algorithm. To be fully precise, a match doesn't
+//! operate on a value, it operates on a place. In certain unsafe circumstances, it is possible for
+//! a place to not contain valid data for its type. This has subtle consequences for empty types.
+//! Take the following:
+//!
+//! ```rust
+//! enum Void {}
+//! let x: u8 = 0;
+//! let ptr: *const Void = &x as *const u8 as *const Void;
+//! unsafe {
+//!     match *ptr {
+//!         _ => println!("Reachable!"),
+//!     }
+//! }
+//! ```
+//!
+//! In this example, `ptr` is a valid pointer pointing to a place with invalid data. The `_` pattern
+//! does not look at the contents of `*ptr`, so this is ok and the arm is taken. In other words,
+//! despite the place we are inspecting being of type `Void`, there is a reachable arm. If the
+//! arm had a binding however:
+//!
+//! ```rust
+//! # #[derive(Copy, Clone)]
+//! # enum Void {}
+//! # let x: u8 = 0;
+//! # let ptr: *const Void = &x as *const u8 as *const Void;
+//! # unsafe {
+//! match *ptr {
+//!     _a => println!("Unreachable!"),
+//! }
+//! # }
+//! ```
+//!
+//! Here the binding loads the value of type `Void` from the `*ptr` place. In this example, this
+//! causes UB since the data is not valid. In the general case, this asserts validity of the data at
+//! `*ptr`. Either way, this arm will never be taken.
+//!
+//! Finally, let's consider the empty match `match *ptr {}`. If we consider this exhaustive, then
+//! having invalid data at `*ptr` is invalid. In other words, the empty match is semantically
+//! equivalent to the `_a => ...` match. In the interest of explicitness, we prefer the case with an
+//! arm, hence we won't tell the user to remove the `_a` arm. In other words, the `_a` arm is
+//! unreachable yet not redundant. This is why we lint on redundant arms rather than unreachable
+//! arms, despite the fact that the lint says "unreachable".
+//!
+//! These considerations only affects certain places, namely those that can contain non-valid data
+//! without UB. These are: pointer dereferences, reference dereferences, and union field accesses.
+//! We track in the algorithm whether a given place is known to contain valid data. This is done
+//! first by inspecting the scrutinee syntactically (which gives us `cx.known_valid_scrutinee`), and
+//! then by tracking validity of each column of the matrix (which correspond to places) as we
+//! recurse into subpatterns. That second part is done through [`ValidityConstraint`], most notably
+//! [`ValidityConstraint::specialize`].
+//!
+//! Having said all that, in practice we don't fully follow what's been presented in this section.
+//! Under `exhaustive_patterns`, we allow omitting empty arms even in `!known_valid` places, for
+//! backwards-compatibility until we have a better alternative. Without `exhaustive_patterns`, we
+//! mostly treat empty types as inhabited, except specifically a non-nested `!` or empty enum. In
+//! this specific case we also allow the empty match regardless of place validity, for
+//! backwards-compatibility. Hopefully we can eventually deprecate this.
+//!
+//!
+//!
 //! # Full example
 //!
 //! We illustrate a full run of the algorithm on the following match.
@@ -348,7 +411,7 @@
 //! ```
 //!
 //! We keep track of the original row for illustration purposes, this is not what the algorithm
-//! actually does (it tracks reachability as a boolean on each row).
+//! actually does (it tracks usefulness as a boolean on each row).
 //!
 //! ```text
 //!  ┐ Patterns:
@@ -377,7 +440,7 @@
 //!  │ │ │ ├─┐ Patterns:
 //!  │ │ │ │ │   1. `[]`
 //!  │ │ │ │ │
-//!  │ │ │ │ │ We note arm 1 is reachable (by `Pair(Some(0), true)`).
+//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), true)`).
 //!  │ │ │ ├─┘
 //!  │ │ │ │
 //!  │ │ │ │ Specialize with `false`:
@@ -385,7 +448,7 @@
 //!  │ │ │ │ │   1. `[]`
 //!  │ │ │ │ │   3. `[]`
 //!  │ │ │ │ │
-//!  │ │ │ │ │ We note arm 1 is reachable (by `Pair(Some(0), false)`).
+//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), false)`).
 //!  │ │ │ ├─┘
 //!  │ │ ├─┘
 //!  │ │ │
@@ -408,7 +471,7 @@
 //!  │ │ │ ├─┐ Patterns:
 //!  │ │ │ │ │   2. `[]`
 //!  │ │ │ │ │
-//!  │ │ │ │ │ We note arm 2 is reachable (by `Pair(Some(1..), false)`).
+//!  │ │ │ │ │ We note arm 2 is useful (by `Pair(Some(1..), false)`).
 //!  │ │ │ ├─┘
 //!  │ │ │ │
 //!  │ │ │ │ Total witnesses for `1..`:
@@ -442,7 +505,7 @@
 //!  │ │ ├─┐ Patterns:
 //!  │ │ │ │   2. `[]`
 //!  │ │ │ │
-//!  │ │ │ │ We note arm 2 is reachable (by `Pair(None, false)`).
+//!  │ │ │ │ We note arm 2 is useful (by `Pair(None, false)`).
 //!  │ │ ├─┘
 //!  │ │ │
 //!  │ │ │ Total witnesses for `None`:
@@ -466,7 +529,7 @@
 //! ```
 //!
 //! We conclude:
-//! - Arm 3 is unreachable (it was never marked as reachable);
+//! - Arm 3 is redundant (it was never marked as useful);
 //! - The match is not exhaustive;
 //! - Adding arms with `Pair(Some(1..), true)` and `Pair(None, true)` would make the match exhaustive.
 //!
@@ -639,13 +702,13 @@ struct MatrixRow<'p, 'tcx> {
     /// Whether the original arm had a guard. This is inherited when specializing.
     is_under_guard: bool,
     /// When we specialize, we remember which row of the original matrix produced a given row of the
-    /// specialized matrix. When we unspecialize, we use this to propagate reachability back up the
+    /// specialized matrix. When we unspecialize, we use this to propagate usefulness back up the
     /// callstack.
     parent_row: usize,
     /// False when the matrix is just built. This is set to `true` by
-    /// [`compute_exhaustiveness_and_reachability`] if the arm is found to be reachable.
+    /// [`compute_exhaustiveness_and_usefulness`] if the arm is found to be useful.
     /// This is reset to `false` when specializing.
-    reachable: bool,
+    useful: bool,
 }
 
 impl<'p, 'tcx> MatrixRow<'p, 'tcx> {
@@ -672,7 +735,7 @@ impl<'p, 'tcx> MatrixRow<'p, 'tcx> {
             pats: patstack,
             parent_row: self.parent_row,
             is_under_guard: self.is_under_guard,
-            reachable: false,
+            useful: false,
         })
     }
 
@@ -688,7 +751,7 @@ impl<'p, 'tcx> MatrixRow<'p, 'tcx> {
             pats: self.pats.pop_head_constructor(pcx, ctor),
             parent_row,
             is_under_guard: self.is_under_guard,
-            reachable: false,
+            useful: false,
         }
     }
 }
@@ -741,7 +804,7 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> {
                 pats: PatStack::from_pattern(arm.pat),
                 parent_row: row_id, // dummy, we won't read it
                 is_under_guard: arm.has_guard,
-                reachable: false,
+                useful: false,
             };
             matrix.expand_and_push(v);
         }
@@ -940,7 +1003,7 @@ impl<'tcx> WitnessStack<'tcx> {
 /// Represents a set of pattern-tuples that are witnesses of non-exhaustiveness for error
 /// reporting. This has similar invariants as `Matrix` does.
 ///
-/// The `WitnessMatrix` returned by [`compute_exhaustiveness_and_reachability`] obeys the invariant
+/// The `WitnessMatrix` returned by [`compute_exhaustiveness_and_usefulness`] obeys the invariant
 /// that the union of the input `Matrix` and the output `WitnessMatrix` together matches the type
 /// exhaustively.
 ///
@@ -1029,7 +1092,7 @@ impl<'tcx> WitnessMatrix<'tcx> {
 /// The core of the algorithm.
 ///
 /// This recursively computes witnesses of the non-exhaustiveness of `matrix` (if any). Also tracks
-/// usefulness of each row in the matrix (in `row.reachable`). We track reachability of each
+/// usefulness of each row in the matrix (in `row.useful`). We track usefulness of each
 /// subpattern using interior mutability in `DeconstructedPat`.
 ///
 /// The input `Matrix` and the output `WitnessMatrix` together match the type exhaustively.
@@ -1038,10 +1101,10 @@ impl<'tcx> WitnessMatrix<'tcx> {
 /// - specialization, where we dig into the rows that have a specific constructor and call ourselves
 ///     recursively;
 /// - unspecialization, where we lift the results from the previous step into results for this step
-///     (using `apply_constructor` and by updating `row.reachable` for each parent row).
+///     (using `apply_constructor` and by updating `row.useful` for each parent row).
 /// This is all explained at the top of the file.
 #[instrument(level = "debug", skip(cx, is_top_level), ret)]
-fn compute_exhaustiveness_and_reachability<'p, 'tcx>(
+fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
     matrix: &mut Matrix<'p, 'tcx>,
     is_top_level: bool,
@@ -1050,10 +1113,10 @@ fn compute_exhaustiveness_and_reachability<'p, 'tcx>(
 
     let Some(ty) = matrix.head_ty() else {
         // The base case: there are no columns in the matrix. We are morally pattern-matching on ().
-        // A row is reachable iff it has no (unguarded) rows above it.
+        // A row is useful iff it has no (unguarded) rows above it.
         for row in matrix.rows_mut() {
-            // All rows are reachable until we find one without a guard.
-            row.reachable = true;
+            // All rows are useful until they're not.
+            row.useful = true;
             if !row.is_under_guard {
                 // There's an unguarded row, so the match is exhaustive, and any subsequent row is
                 // unreachable.
@@ -1095,7 +1158,7 @@ fn compute_exhaustiveness_and_reachability<'p, 'tcx>(
         // Dig into rows that match `ctor`.
         let mut spec_matrix = matrix.specialize_constructor(pcx, &ctor);
         let mut witnesses = ensure_sufficient_stack(|| {
-            compute_exhaustiveness_and_reachability(cx, &mut spec_matrix, false)
+            compute_exhaustiveness_and_usefulness(cx, &mut spec_matrix, false)
         });
 
         if !only_report_missing || matches!(ctor, Constructor::Missing) {
@@ -1113,14 +1176,14 @@ fn compute_exhaustiveness_and_reachability<'p, 'tcx>(
         // A parent row is useful if any of its children is.
         for child_row in spec_matrix.rows() {
             let parent_row = &mut matrix.rows[child_row.parent_row];
-            parent_row.reachable = parent_row.reachable || child_row.reachable;
+            parent_row.useful = parent_row.useful || child_row.useful;
         }
     }
 
-    // Record that the subpattern is reachable.
+    // Record usefulness in the patterns.
     for row in matrix.rows() {
-        if row.reachable {
-            row.head().set_reachable();
+        if row.useful {
+            row.head().set_useful();
         }
     }
 
@@ -1130,8 +1193,8 @@ fn compute_exhaustiveness_and_reachability<'p, 'tcx>(
 /// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
 /// inspect the same subvalue/place".
 /// This is used to traverse patterns column-by-column for lints. Despite similarities with
-/// [`compute_exhaustiveness_and_reachability`], this does a different traversal. Notably this is
-/// linear in the depth of patterns, whereas `compute_exhaustiveness_and_reachability` is worst-case
+/// [`compute_exhaustiveness_and_usefulness`], this does a different traversal. Notably this is
+/// linear in the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case
 /// exponential (exhaustiveness is NP-complete). The core difference is that we treat sub-columns
 /// separately.
 ///
@@ -1351,28 +1414,29 @@ pub(crate) struct MatchArm<'p, 'tcx> {
     pub(crate) has_guard: bool,
 }
 
-/// Indicates whether or not a given arm is reachable.
+/// Indicates whether or not a given arm is useful.
 #[derive(Clone, Debug)]
-pub(crate) enum Reachability {
-    /// The arm is reachable. This additionally carries a set of or-pattern branches that have been
-    /// found to be unreachable despite the overall arm being reachable. Used only in the presence
-    /// of or-patterns, otherwise it stays empty.
-    Reachable(Vec<Span>),
-    /// The arm is unreachable.
-    Unreachable,
+pub(crate) enum Usefulness {
+    /// The arm is useful. This additionally carries a set of or-pattern branches that have been
+    /// found to be redundant despite the overall arm being useful. Used only in the presence of
+    /// or-patterns, otherwise it stays empty.
+    Useful(Vec<Span>),
+    /// The arm is redundant and can be removed without changing the behavior of the match
+    /// expression.
+    Redundant,
 }
 
-/// The output of checking a match for exhaustiveness and arm reachability.
+/// The output of checking a match for exhaustiveness and arm usefulness.
 pub(crate) struct UsefulnessReport<'p, 'tcx> {
-    /// For each arm of the input, whether that arm is reachable after the arms above it.
-    pub(crate) arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Reachability)>,
+    /// For each arm of the input, whether that arm is useful after the arms above it.
+    pub(crate) arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Usefulness)>,
     /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of
     /// exhaustiveness.
     pub(crate) non_exhaustiveness_witnesses: Vec<WitnessPat<'tcx>>,
 }
 
 /// The entrypoint for this file. Computes whether a match is exhaustive and which of its arms are
-/// reachable.
+/// useful.
 #[instrument(skip(cx, arms), level = "debug")]
 pub(crate) fn compute_match_usefulness<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
@@ -1380,8 +1444,7 @@ pub(crate) fn compute_match_usefulness<'p, 'tcx>(
     scrut_ty: Ty<'tcx>,
 ) -> UsefulnessReport<'p, 'tcx> {
     let mut matrix = Matrix::new(cx, arms, scrut_ty);
-    let non_exhaustiveness_witnesses =
-        compute_exhaustiveness_and_reachability(cx, &mut matrix, true);
+    let non_exhaustiveness_witnesses = compute_exhaustiveness_and_usefulness(cx, &mut matrix, true);
 
     let non_exhaustiveness_witnesses: Vec<_> = non_exhaustiveness_witnesses.single_column();
     let arm_usefulness: Vec<_> = arms
@@ -1389,12 +1452,13 @@ pub(crate) fn compute_match_usefulness<'p, 'tcx>(
         .copied()
         .map(|arm| {
             debug!(?arm);
-            let reachability = if arm.pat.is_reachable() {
-                Reachability::Reachable(arm.pat.unreachable_spans())
+            // We warn when a pattern is not useful.
+            let usefulness = if arm.pat.is_useful() {
+                Usefulness::Useful(arm.pat.redundant_spans())
             } else {
-                Reachability::Unreachable
+                Usefulness::Redundant
             };
-            (arm, reachability)
+            (arm, usefulness)
         })
         .collect();
     let report = UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses };
-- 
cgit 1.4.1-3-g733a5


From 2186f98f16242545e09a4996e675738e3bf9209b Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sun, 29 Oct 2023 19:18:18 +0100
Subject: Correctly handle empty constructors

- `ConstructorSet` knows about both empty and nonempty constructors;
- If an empty constructor is present in the column, we output it in
    `split().present`.
---
 .../src/thir/pattern/deconstruct_pat.rs            | 364 ++++++++++++---------
 .../rustc_mir_build/src/thir/pattern/usefulness.rs |   6 +-
 2 files changed, 212 insertions(+), 158 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 094acbc3f8a..f1ad0f25cd2 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -10,6 +10,7 @@
 //! precisely here.
 //!
 //!
+//!
 //! # Constructor grouping and splitting
 //!
 //! As explained in the corresponding section in [`super::usefulness`], to make usefulness tractable
@@ -49,6 +50,7 @@
 //! [`IntRange::split`]) and slice splitting (see [`Slice::split`]).
 //!
 //!
+//!
 //! # The `Missing` constructor
 //!
 //! We detail a special case of constructor splitting that is a bit subtle. Take the following:
@@ -77,6 +79,69 @@
 //!
 //!
 //!
+//! ## Empty types, empty constructors, and the `exhaustive_patterns` feature
+//!
+//! An empty type is a type that has no valid value, like `!`, `enum Void {}`, or `Result<!, !>`.
+//! They require careful handling.
+//!
+//! First, for soundness reasons related to the possible existence of invalid values, by default we
+//! don't treat empty types as empty. We force them to be matched with wildcards. Except if the
+//! `exhaustive_patterns` feature is turned on, in which case we do treat them as empty. And also
+//! except if the type has no constructors (like `enum Void {}` but not like `Result<!, !>`), we
+//! specifically allow `match void {}` to be exhaustive. There are additionally considerations of
+//! place validity that are handled in `super::usefulness`. Yes this is a bit tricky.
+//!
+//! The second thing is that regardless of the above, it is always allowed to use all the
+//! constructors of a type. For example, all the following is ok:
+//!
+//! ```rust,ignore(example)
+//! # #![feature(never_type)]
+//! # #![feature(exhaustive_patterns)]
+//! fn foo(x: Option<!>) {
+//!   match x {
+//!     None => {}
+//!     Some(_) => {}
+//!   }
+//! }
+//! fn bar(x: &[!]) -> u32 {
+//!   match x {
+//!     [] => 1,
+//!     [_] => 2,
+//!     [_, _] => 3,
+//!   }
+//! }
+//! ```
+//!
+//! Moreover, take the following:
+//!
+//! ```rust
+//! # #![feature(never_type)]
+//! # #![feature(exhaustive_patterns)]
+//! # let x = None::<!>;
+//! match x {
+//!   None => {}
+//! }
+//! ```
+//!
+//! On a normal type, we would identify `Some` as missing and tell the user. If `x: Option<!>`
+//! however (and `exhaustive_patterns` is on), it's ok to omit `Some`. When listing the constructors
+//! of a type, we must therefore track which can be omitted.
+//!
+//! Let's call "empty" a constructor that matches no valid value for the type, like `Some` for the
+//! type `Option<!>`. What this all means is that `ConstructorSet` must know which constructors are
+//! empty. The difference between empty and nonempty constructors is that empty constructors need
+//! not be present for the match to be exhaustive.
+//!
+//! A final remark: empty constructors of arity 0 break specialization, we must avoid them. The
+//! reason is that if we specialize by them, nothing remains to witness the emptiness; the rest of
+//! the algorithm can't distinguish them from a nonempty constructor. The only known case where this
+//! could happen is the `[..]` pattern on `[!; N]` with `N > 0` so we must take care to not emit it.
+//!
+//! This is all handled by [`ConstructorSet::for_ty`] and [`ConstructorSet::split`]. The invariants
+//! of [`SplitConstructorSet`] are also of interest.
+//!
+//!
+//!
 //! ## Opaque patterns
 //!
 //! Some patterns, such as constants that are not allowed to be matched structurally, cannot be
@@ -95,7 +160,7 @@ use rustc_apfloat::ieee::{DoubleS, IeeeFloat, SingleS};
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::RangeEnd;
-use rustc_index::Idx;
+use rustc_index::{Idx, IndexVec};
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::Scalar;
@@ -482,8 +547,12 @@ pub(super) struct Slice {
 impl Slice {
     fn new(array_len: Option<usize>, kind: SliceKind) -> Self {
         let kind = match (array_len, kind) {
-            // If the middle `..` is empty, we effectively have a fixed-length pattern.
-            (Some(len), VarLen(prefix, suffix)) if prefix + suffix >= len => FixedLen(len),
+            // If the middle `..` has length 0, we effectively have a fixed-length pattern.
+            (Some(len), VarLen(prefix, suffix)) if prefix + suffix == len => FixedLen(len),
+            (Some(len), VarLen(prefix, suffix)) if prefix + suffix > len => bug!(
+                "Slice pattern of length {} longer than its array length {len}",
+                prefix + suffix
+            ),
             _ => kind,
         };
         Slice { array_len, kind }
@@ -583,17 +652,18 @@ impl Slice {
         let mut seen_fixed_lens = FxHashSet::default();
         match &mut max_slice {
             VarLen(max_prefix_len, max_suffix_len) => {
+                // A length larger than any fixed-length slice encountered.
+                // We start at 1 in case the subtype is empty because in that case the zero-length
+                // slice must be treated separately from the rest.
+                let mut fixed_len_upper_bound = 1;
                 // We grow `max_slice` to be larger than all slices encountered, as described above.
-                // For diagnostics, we keep the prefix and suffix lengths separate, but grow them so that
-                // `L = max_prefix_len + max_suffix_len`.
-                let mut max_fixed_len = 0;
+                // `L` is `max_slice.arity()`. For diagnostics, we keep the prefix and suffix
+                // lengths separate.
                 for slice in column_slices {
                     match slice.kind {
                         FixedLen(len) => {
-                            max_fixed_len = cmp::max(max_fixed_len, len);
-                            if arity <= len {
-                                seen_fixed_lens.insert(len);
-                            }
+                            fixed_len_upper_bound = cmp::max(fixed_len_upper_bound, len + 1);
+                            seen_fixed_lens.insert(len);
                         }
                         VarLen(prefix, suffix) => {
                             *max_prefix_len = cmp::max(*max_prefix_len, prefix);
@@ -602,12 +672,11 @@ impl Slice {
                         }
                     }
                 }
-                // We want `L = max(L, max_fixed_len + 1)`, modulo the fact that we keep prefix and
-                // suffix separate.
-                if max_fixed_len + 1 >= *max_prefix_len + *max_suffix_len {
-                    // The subtraction can't overflow thanks to the above check.
-                    // The new `max_prefix_len` is larger than its previous value.
-                    *max_prefix_len = max_fixed_len + 1 - *max_suffix_len;
+                // If `fixed_len_upper_bound >= L`, we set `L` to `fixed_len_upper_bound`.
+                if let Some(delta) =
+                    fixed_len_upper_bound.checked_sub(*max_prefix_len + *max_suffix_len)
+                {
+                    *max_prefix_len += delta
                 }
 
                 // We cap the arity of `max_slice` at the array size.
@@ -856,34 +925,46 @@ impl<'tcx> Constructor<'tcx> {
     }
 }
 
-/// Describes the set of all constructors for a type.
+#[derive(Debug, Clone, Copy)]
+pub(super) enum VariantVisibility {
+    /// Variant that doesn't fit the other cases, i.e. most variants.
+    Visible,
+    /// Variant behind an unstable gate or with the `#[doc(hidden)]` attribute. It will not be
+    /// mentioned in diagnostics unless the user mentioned it first.
+    Hidden,
+    /// Variant that matches no value. E.g. `Some::<Option<!>>` if the `exhaustive_patterns` feature
+    /// is enabled. Like `Hidden`, it will not be mentioned in diagnostics unless the user mentioned
+    /// it first.
+    Empty,
+}
+
+/// Describes the set of all constructors for a type. For details, in particular about the emptiness
+/// of constructors, see the top of the file.
+///
+/// In terms of division of responsibility, [`ConstructorSet::split`] handles all of the
+/// `exhaustive_patterns` feature.
 #[derive(Debug)]
 pub(super) enum ConstructorSet {
-    /// The type has a single constructor, e.g. `&T` or a struct.
-    Single,
-    /// This type has the following list of constructors.
-    /// Some variants are hidden, which means they won't be mentioned in diagnostics unless the user
-    /// mentioned them first. We use this for variants behind an unstable gate as well as
-    /// `#[doc(hidden)]` ones.
-    Variants {
-        visible_variants: Vec<VariantIdx>,
-        hidden_variants: Vec<VariantIdx>,
-        non_exhaustive: bool,
-    },
+    /// The type has a single constructor, e.g. `&T` or a struct. `empty` tracks whether the
+    /// constructor is empty.
+    Single { empty: bool },
+    /// This type has the following list of constructors. If `variants` is empty and
+    /// `non_exhaustive` is false, don't use this; use `NoConstructors` instead.
+    Variants { variants: IndexVec<VariantIdx, VariantVisibility>, non_exhaustive: bool },
     /// Booleans.
     Bool,
     /// The type is spanned by integer values. The range or ranges give the set of allowed values.
     /// The second range is only useful for `char`.
     Integers { range_1: IntRange, range_2: Option<IntRange> },
-    /// The type is matched by slices. The usize is the compile-time length of the array, if known.
-    Slice(Option<usize>),
-    /// The type is matched by slices whose elements are uninhabited.
-    SliceOfEmpty,
+    /// The type is matched by slices. `array_len` is the compile-time length of the array, if
+    /// known. If `subtype_is_empty`, all constructors are empty except possibly the zero-length
+    /// slice `[]`.
+    Slice { array_len: Option<usize>, subtype_is_empty: bool },
     /// The constructors cannot be listed, and the type cannot be matched exhaustively. E.g. `str`,
     /// floats.
     Unlistable,
-    /// The type has no inhabitants.
-    Uninhabited,
+    /// The type has no constructors (not even empty ones). This is `!` and empty enums.
+    NoConstructors,
 }
 
 /// Describes the result of analyzing the constructors in a column of a match.
@@ -913,6 +994,8 @@ pub(super) struct SplitConstructorSet<'tcx> {
 
 impl ConstructorSet {
     /// Creates a set that represents all the constructors of `ty`.
+    ///
+    /// See at the top of the file for considerations of emptiness.
     #[instrument(level = "debug", skip(cx), ret)]
     pub(super) fn for_ty<'p, 'tcx>(cx: &MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
         let make_range = |start, end| {
@@ -924,12 +1007,6 @@ impl ConstructorSet {
         };
         // This determines the set of all possible constructors for the type `ty`. For numbers,
         // arrays and slices we use ranges and variable-length slices when appropriate.
-        //
-        // If the `exhaustive_patterns` feature is enabled, we make sure to omit constructors that
-        // are statically impossible. E.g., for `Option<!>`, we do not include `Some(_)` in the
-        // returned list of constructors.
-        // Invariant: this is `Uninhabited` if and only if the type is uninhabited (as determined by
-        // `cx.is_uninhabited()`).
         match ty.kind() {
             ty::Bool => Self::Bool,
             ty::Char => {
@@ -963,82 +1040,74 @@ impl ConstructorSet {
                 };
                 Self::Integers { range_1: range, range_2: None }
             }
-            ty::Array(sub_ty, len) if len.try_eval_target_usize(cx.tcx, cx.param_env).is_some() => {
-                let len = len.eval_target_usize(cx.tcx, cx.param_env) as usize;
-                if len != 0 && cx.is_uninhabited(*sub_ty) {
-                    Self::Uninhabited
-                } else {
-                    Self::Slice(Some(len))
-                }
+            ty::Slice(sub_ty) => {
+                Self::Slice { array_len: None, subtype_is_empty: cx.is_uninhabited(*sub_ty) }
             }
-            // Treat arrays of a constant but unknown length like slices.
-            ty::Array(sub_ty, _) | ty::Slice(sub_ty) => {
-                if cx.is_uninhabited(*sub_ty) {
-                    Self::SliceOfEmpty
-                } else {
-                    Self::Slice(None)
+            ty::Array(sub_ty, len) => {
+                // We treat arrays of a constant but unknown length like slices.
+                Self::Slice {
+                    array_len: len.try_eval_target_usize(cx.tcx, cx.param_env).map(|l| l as usize),
+                    subtype_is_empty: cx.is_uninhabited(*sub_ty),
                 }
             }
             ty::Adt(def, args) if def.is_enum() => {
-                // If the enum is declared as `#[non_exhaustive]`, we treat it as if it had an
-                // additional "unknown" constructor.
-                // There is no point in enumerating all possible variants, because the user can't
-                // actually match against them all themselves. So we always return only the fictitious
-                // constructor.
-                // E.g., in an example like:
-                //
-                // ```
-                //     let err: io::ErrorKind = ...;
-                //     match err {
-                //         io::ErrorKind::NotFound => {},
-                //     }
-                // ```
-                //
-                // we don't want to show every possible IO error, but instead have only `_` as the
-                // witness.
                 let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(ty);
-
                 if def.variants().is_empty() && !is_declared_nonexhaustive {
-                    Self::Uninhabited
+                    Self::NoConstructors
                 } else {
-                    let is_exhaustive_pat_feature = cx.tcx.features().exhaustive_patterns;
-                    let (hidden_variants, visible_variants) = def
-                        .variants()
-                        .iter_enumerated()
-                        .filter(|(_, v)| {
-                            // If `exhaustive_patterns` is enabled, we exclude variants known to be
-                            // uninhabited.
-                            !is_exhaustive_pat_feature
-                                || v.inhabited_predicate(cx.tcx, *def)
-                                    .instantiate(cx.tcx, args)
-                                    .apply(cx.tcx, cx.param_env, cx.module)
-                        })
-                        .map(|(idx, _)| idx)
-                        .partition(|idx| {
-                            let variant_def_id = def.variant(*idx).def_id;
-                            // Filter variants that depend on a disabled unstable feature.
-                            let is_unstable = matches!(
-                                cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
-                                EvalResult::Deny { .. }
-                            );
-                            // Filter foreign `#[doc(hidden)]` variants.
-                            let is_doc_hidden =
-                                cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
-                            is_unstable || is_doc_hidden
-                        });
-
-                    Self::Variants {
-                        visible_variants,
-                        hidden_variants,
-                        non_exhaustive: is_declared_nonexhaustive,
+                    let mut variants =
+                        IndexVec::from_elem(VariantVisibility::Visible, def.variants());
+                    for (idx, v) in def.variants().iter_enumerated() {
+                        let variant_def_id = def.variant(idx).def_id;
+                        // Visibly uninhabited variants.
+                        let is_inhabited = v
+                            .inhabited_predicate(cx.tcx, *def)
+                            .instantiate(cx.tcx, args)
+                            .apply(cx.tcx, cx.param_env, cx.module);
+                        // Variants that depend on a disabled unstable feature.
+                        let is_unstable = matches!(
+                            cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
+                            EvalResult::Deny { .. }
+                        );
+                        // Foreign `#[doc(hidden)]` variants.
+                        let is_doc_hidden =
+                            cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
+                        let visibility = if !is_inhabited {
+                            // FIXME: handle empty+hidden
+                            VariantVisibility::Empty
+                        } else if is_unstable || is_doc_hidden {
+                            VariantVisibility::Hidden
+                        } else {
+                            VariantVisibility::Visible
+                        };
+                        variants[idx] = visibility;
                     }
+
+                    Self::Variants { variants, non_exhaustive: is_declared_nonexhaustive }
                 }
             }
-            ty::Never => Self::Uninhabited,
-            _ if cx.is_uninhabited(ty) => Self::Uninhabited,
-            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => Self::Single,
+            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => {
+                Self::Single { empty: cx.is_uninhabited(ty) }
+            }
+            ty::Never => Self::NoConstructors,
             // This type is one for which we cannot list constructors, like `str` or `f64`.
-            _ => Self::Unlistable,
+            // FIXME(Nadrieril): which of these are actually allowed?
+            ty::Float(_)
+            | ty::Str
+            | ty::Foreign(_)
+            | ty::RawPtr(_)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Closure(_, _)
+            | ty::Coroutine(_, _, _)
+            | ty::CoroutineWitness(_, _)
+            | ty::Alias(_, _)
+            | ty::Param(_)
+            | ty::Bound(_, _)
+            | ty::Placeholder(_)
+            | ty::Infer(_)
+            | ty::Error(_) => Self::Unlistable,
         }
     }
 
@@ -1056,6 +1125,9 @@ impl ConstructorSet {
         'tcx: 'a,
     {
         let mut present: SmallVec<[_; 1]> = SmallVec::new();
+        // Empty constructors found missing.
+        let mut missing_empty = Vec::new();
+        // Nonempty constructors found missing.
         let mut missing = Vec::new();
         // Constructors in `ctors`, except wildcards and opaques.
         let mut seen = Vec::new();
@@ -1068,38 +1140,36 @@ impl ConstructorSet {
         }
 
         match self {
-            ConstructorSet::Single => {
-                if seen.is_empty() {
-                    missing.push(Single);
-                } else {
+            ConstructorSet::Single { empty } => {
+                if !seen.is_empty() {
                     present.push(Single);
+                } else if *empty {
+                    missing_empty.push(Single);
+                } else {
+                    missing.push(Single);
                 }
             }
-            ConstructorSet::Variants { visible_variants, hidden_variants, non_exhaustive } => {
+            ConstructorSet::Variants { variants, non_exhaustive } => {
                 let seen_set: FxHashSet<_> = seen.iter().map(|c| c.as_variant().unwrap()).collect();
                 let mut skipped_a_hidden_variant = false;
 
-                for variant in visible_variants {
-                    let ctor = Variant(*variant);
-                    if seen_set.contains(variant) {
+                for (idx, visibility) in variants.iter_enumerated() {
+                    let ctor = Variant(idx);
+                    if seen_set.contains(&idx) {
                         present.push(ctor);
                     } else {
-                        missing.push(ctor);
+                        // We only put visible variants directly into `missing`.
+                        match visibility {
+                            VariantVisibility::Visible => missing.push(ctor),
+                            VariantVisibility::Hidden => skipped_a_hidden_variant = true,
+                            VariantVisibility::Empty => missing_empty.push(ctor),
+                        }
                     }
                 }
 
-                for variant in hidden_variants {
-                    let ctor = Variant(*variant);
-                    if seen_set.contains(variant) {
-                        present.push(ctor);
-                    } else {
-                        skipped_a_hidden_variant = true;
-                    }
-                }
                 if skipped_a_hidden_variant {
                     missing.push(Hidden);
                 }
-
                 if *non_exhaustive {
                     missing.push(NonExhaustive);
                 }
@@ -1143,33 +1213,22 @@ impl ConstructorSet {
                     }
                 }
             }
-            &ConstructorSet::Slice(array_len) => {
+            ConstructorSet::Slice { array_len, subtype_is_empty } => {
                 let seen_slices = seen.iter().map(|c| c.as_slice().unwrap());
-                let base_slice = Slice::new(array_len, VarLen(0, 0));
+                let base_slice = Slice::new(*array_len, VarLen(0, 0));
                 for (seen, splitted_slice) in base_slice.split(seen_slices) {
                     let ctor = Slice(splitted_slice);
                     match seen {
-                        Presence::Unseen => missing.push(ctor),
                         Presence::Seen => present.push(ctor),
-                    }
-                }
-            }
-            ConstructorSet::SliceOfEmpty => {
-                // This one is tricky because even though there's only one possible value of this
-                // type (namely `[]`), slice patterns of all lengths are allowed, they're just
-                // unreachable if length != 0.
-                // We still gather the seen constructors in `present`, but the only slice that can
-                // go in `missing` is `[]`.
-                let seen_slices = seen.iter().map(|c| c.as_slice().unwrap());
-                let base_slice = Slice::new(None, VarLen(0, 0));
-                for (seen, splitted_slice) in base_slice.split(seen_slices) {
-                    let ctor = Slice(splitted_slice);
-                    match seen {
-                        Presence::Seen => present.push(ctor),
-                        Presence::Unseen if splitted_slice.arity() == 0 => {
-                            missing.push(Slice(Slice::new(None, FixedLen(0))))
+                        Presence::Unseen => {
+                            if *subtype_is_empty && splitted_slice.arity() != 0 {
+                                // We have subpatterns of an empty type, so the constructor is
+                                // empty.
+                                missing_empty.push(ctor);
+                            } else {
+                                missing.push(ctor);
+                            }
                         }
-                        Presence::Unseen => {}
                     }
                 }
             }
@@ -1179,17 +1238,16 @@ impl ConstructorSet {
                 present.extend(seen);
                 missing.push(NonExhaustive);
             }
-            // If `exhaustive_patterns` is disabled and our scrutinee is an empty type, we cannot
-            // expose its emptiness. The exception is if the pattern is at the top level, because we
-            // want empty matches to be considered exhaustive.
-            ConstructorSet::Uninhabited
-                if !pcx.cx.tcx.features().exhaustive_patterns && !pcx.is_top_level =>
-            {
-                missing.push(NonExhaustive);
+            ConstructorSet::NoConstructors => {
+                if !pcx.is_top_level {
+                    missing_empty.push(NonExhaustive);
+                }
             }
-            ConstructorSet::Uninhabited => {}
         }
 
+        if !pcx.cx.tcx.features().exhaustive_patterns {
+            missing.extend(missing_empty);
+        }
         SplitConstructorSet { present, missing }
     }
 }
@@ -1263,7 +1321,7 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
             // `field.ty()` doesn't normalize after substituting.
             let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
             let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
-            let is_uninhabited = cx.is_uninhabited(ty);
+            let is_uninhabited = cx.tcx.features().exhaustive_patterns && cx.is_uninhabited(ty);
 
             if is_uninhabited && (!is_visible || is_non_exhaustive) {
                 None
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index e77f158a980..38a66522ac6 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -596,11 +596,7 @@ pub(crate) struct MatchCheckCtxt<'p, 'tcx> {
 
 impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
     pub(super) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
-        if self.tcx.features().exhaustive_patterns {
-            !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
-        } else {
-            false
-        }
+        !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
     }
 
     /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
-- 
cgit 1.4.1-3-g733a5


From 1978168c136ba65b8da3f5a45834631c13d7c56f Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sun, 26 Nov 2023 12:54:52 +0100
Subject: Detect cycles in `InhabitedPredicate::apply`

This is for post-monomorphization cycles. These are only caught later
(in drop elaboration for the example that I saw), so we need to handle
them here.

This issue wasn't noticed before because exhaustiveness only checked
inhabitedness when `exhaustive_patterns` was on. The preceding commit
now check inhabitedness always, which revealed the problem.
---
 .../src/ty/inhabitedness/inhabited_predicate.rs    | 40 ++++++++++++++++------
 compiler/rustc_middle/src/ty/inhabitedness/mod.rs  |  1 +
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
index f278cace99d..ae17942785f 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
@@ -1,3 +1,5 @@
+use smallvec::SmallVec;
+
 use crate::ty::context::TyCtxt;
 use crate::ty::{self, DefId, ParamEnv, Ty};
 
@@ -31,27 +33,31 @@ impl<'tcx> InhabitedPredicate<'tcx> {
     /// Returns true if the corresponding type is inhabited in the given
     /// `ParamEnv` and module
     pub fn apply(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, module_def_id: DefId) -> bool {
-        let Ok(result) = self
-            .apply_inner::<!>(tcx, param_env, &|id| Ok(tcx.is_descendant_of(module_def_id, id)));
+        let Ok(result) = self.apply_inner::<!>(tcx, param_env, &mut Default::default(), &|id| {
+            Ok(tcx.is_descendant_of(module_def_id, id))
+        });
         result
     }
 
     /// Same as `apply`, but returns `None` if self contains a module predicate
     pub fn apply_any_module(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> Option<bool> {
-        self.apply_inner(tcx, param_env, &|_| Err(())).ok()
+        self.apply_inner(tcx, param_env, &mut Default::default(), &|_| Err(())).ok()
     }
 
     /// Same as `apply`, but `NotInModule(_)` predicates yield `false`. That is,
     /// privately uninhabited types are considered always uninhabited.
     pub fn apply_ignore_module(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> bool {
-        let Ok(result) = self.apply_inner::<!>(tcx, param_env, &|_| Ok(true));
+        let Ok(result) =
+            self.apply_inner::<!>(tcx, param_env, &mut Default::default(), &|_| Ok(true));
         result
     }
 
-    fn apply_inner<E>(
+    #[instrument(level = "debug", skip(tcx, param_env, in_module), ret)]
+    fn apply_inner<E: std::fmt::Debug>(
         self,
         tcx: TyCtxt<'tcx>,
         param_env: ParamEnv<'tcx>,
+        eval_stack: &mut SmallVec<[Ty<'tcx>; 1]>, // for cycle detection
         in_module: &impl Fn(DefId) -> Result<bool, E>,
     ) -> Result<bool, E> {
         match self {
@@ -71,11 +77,25 @@ impl<'tcx> InhabitedPredicate<'tcx> {
                 match normalized_pred {
                     // We don't have more information than we started with, so consider inhabited.
                     Self::GenericType(_) => Ok(true),
-                    pred => pred.apply_inner(tcx, param_env, in_module),
+                    pred => {
+                        // A type which is cyclic when monomorphized can happen here since the
+                        // layout error would only trigger later. See e.g. `tests/ui/sized/recursive-type-2.rs`.
+                        if eval_stack.contains(&t) {
+                            return Ok(true); // Recover; this will error later.
+                        }
+                        eval_stack.push(t);
+                        let ret = pred.apply_inner(tcx, param_env, eval_stack, in_module);
+                        eval_stack.pop();
+                        ret
+                    }
                 }
             }
-            Self::And([a, b]) => try_and(a, b, |x| x.apply_inner(tcx, param_env, in_module)),
-            Self::Or([a, b]) => try_or(a, b, |x| x.apply_inner(tcx, param_env, in_module)),
+            Self::And([a, b]) => {
+                try_and(a, b, |x| x.apply_inner(tcx, param_env, eval_stack, in_module))
+            }
+            Self::Or([a, b]) => {
+                try_or(a, b, |x| x.apply_inner(tcx, param_env, eval_stack, in_module))
+            }
         }
     }
 
@@ -197,7 +217,7 @@ impl<'tcx> InhabitedPredicate<'tcx> {
 
 // this is basically like `f(a)? && f(b)?` but different in the case of
 // `Ok(false) && Err(_) -> Ok(false)`
-fn try_and<T, E>(a: T, b: T, f: impl Fn(T) -> Result<bool, E>) -> Result<bool, E> {
+fn try_and<T, E>(a: T, b: T, mut f: impl FnMut(T) -> Result<bool, E>) -> Result<bool, E> {
     let a = f(a);
     if matches!(a, Ok(false)) {
         return Ok(false);
@@ -209,7 +229,7 @@ fn try_and<T, E>(a: T, b: T, f: impl Fn(T) -> Result<bool, E>) -> Result<bool, E
     }
 }
 
-fn try_or<T, E>(a: T, b: T, f: impl Fn(T) -> Result<bool, E>) -> Result<bool, E> {
+fn try_or<T, E>(a: T, b: T, mut f: impl FnMut(T) -> Result<bool, E>) -> Result<bool, E> {
     let a = f(a);
     if matches!(a, Ok(true)) {
         return Ok(true);
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 68ac54e899a..d67fb9fd0b7 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -103,6 +103,7 @@ impl<'tcx> VariantDef {
 }
 
 impl<'tcx> Ty<'tcx> {
+    #[instrument(level = "debug", skip(tcx), ret)]
     pub fn inhabited_predicate(self, tcx: TyCtxt<'tcx>) -> InhabitedPredicate<'tcx> {
         match self.kind() {
             // For now, unions are always considered inhabited
-- 
cgit 1.4.1-3-g733a5


From f5dbb54648e40f3363321cd5ea6cd903db1af994 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 18 Nov 2023 03:52:54 +0100
Subject: Track place validity

---
 .../src/thir/pattern/check_match.rs                | 124 ++++++++++++++++++---
 .../rustc_mir_build/src/thir/pattern/usefulness.rs | 122 +++++++++++++++++---
 2 files changed, 216 insertions(+), 30 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 89b7b3619be..41510d31530 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -42,7 +42,7 @@ pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(), Err
 
     for param in thir.params.iter() {
         if let Some(box ref pattern) = param.pat {
-            visitor.check_binding_is_irrefutable(pattern, "function argument", None);
+            visitor.check_binding_is_irrefutable(pattern, "function argument", None, None);
         }
     }
     visitor.error
@@ -254,10 +254,11 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
                 self.with_lint_level(lint_level, |this| this.visit_land_rhs(&this.thir[value]))
             }
             ExprKind::Let { box ref pat, expr } => {
+                let expr = &self.thir()[expr];
                 self.with_let_source(LetSource::None, |this| {
-                    this.visit_expr(&this.thir()[expr]);
+                    this.visit_expr(expr);
                 });
-                Ok(Some((ex.span, self.is_let_irrefutable(pat)?)))
+                Ok(Some((ex.span, self.is_let_irrefutable(pat, Some(expr))?)))
             }
             _ => {
                 self.with_let_source(LetSource::None, |this| {
@@ -287,35 +288,114 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         }
     }
 
+    /// Inspects the match scrutinee expression to determine whether the place it evaluates to may
+    /// hold invalid data.
+    fn is_known_valid_scrutinee(&self, scrutinee: &Expr<'tcx>) -> bool {
+        use ExprKind::*;
+        match &scrutinee.kind {
+            // Both pointers and references can validly point to a place with invalid data.
+            Deref { .. } => false,
+            // Inherit validity of the parent place, unless the parent is an union.
+            Field { lhs, .. } => {
+                let lhs = &self.thir()[*lhs];
+                match lhs.ty.kind() {
+                    ty::Adt(def, _) if def.is_union() => false,
+                    _ => self.is_known_valid_scrutinee(lhs),
+                }
+            }
+            // Essentially a field access.
+            Index { lhs, .. } => {
+                let lhs = &self.thir()[*lhs];
+                self.is_known_valid_scrutinee(lhs)
+            }
+
+            // No-op.
+            Scope { value, .. } => self.is_known_valid_scrutinee(&self.thir()[*value]),
+
+            // Casts don't cause a load.
+            NeverToAny { source }
+            | Cast { source }
+            | Use { source }
+            | PointerCoercion { source, .. }
+            | PlaceTypeAscription { source, .. }
+            | ValueTypeAscription { source, .. } => {
+                self.is_known_valid_scrutinee(&self.thir()[*source])
+            }
+
+            // These diverge.
+            Become { .. } | Break { .. } | Continue { .. } | Return { .. } => true,
+
+            // These are statements that evaluate to `()`.
+            Assign { .. } | AssignOp { .. } | InlineAsm { .. } | Let { .. } => true,
+
+            // These evaluate to a value.
+            AddressOf { .. }
+            | Adt { .. }
+            | Array { .. }
+            | Binary { .. }
+            | Block { .. }
+            | Borrow { .. }
+            | Box { .. }
+            | Call { .. }
+            | Closure { .. }
+            | ConstBlock { .. }
+            | ConstParam { .. }
+            | If { .. }
+            | Literal { .. }
+            | LogicalOp { .. }
+            | Loop { .. }
+            | Match { .. }
+            | NamedConst { .. }
+            | NonHirLiteral { .. }
+            | OffsetOf { .. }
+            | Repeat { .. }
+            | StaticRef { .. }
+            | ThreadLocalRef { .. }
+            | Tuple { .. }
+            | Unary { .. }
+            | UpvarRef { .. }
+            | VarRef { .. }
+            | ZstLiteral { .. }
+            | Yield { .. } => true,
+        }
+    }
+
     fn new_cx(
         &self,
         refutability: RefutableFlag,
-        match_span: Option<Span>,
+        whole_match_span: Option<Span>,
+        scrutinee: Option<&Expr<'tcx>>,
         scrut_span: Span,
     ) -> MatchCheckCtxt<'p, 'tcx> {
         let refutable = match refutability {
             Irrefutable => false,
             Refutable => true,
         };
+        // If we don't have a scrutinee we're either a function parameter or a `let x;`. Both cases
+        // require validity.
+        let known_valid_scrutinee =
+            scrutinee.map(|scrut| self.is_known_valid_scrutinee(scrut)).unwrap_or(true);
         MatchCheckCtxt {
             tcx: self.tcx,
             param_env: self.param_env,
             module: self.tcx.parent_module(self.lint_level).to_def_id(),
             pattern_arena: self.pattern_arena,
             match_lint_level: self.lint_level,
-            match_span,
+            whole_match_span,
             scrut_span,
             refutable,
+            known_valid_scrutinee,
         }
     }
 
     #[instrument(level = "trace", skip(self))]
     fn check_let(&mut self, pat: &Pat<'tcx>, scrutinee: Option<ExprId>, span: Span) {
         assert!(self.let_source != LetSource::None);
+        let scrut = scrutinee.map(|id| &self.thir[id]);
         if let LetSource::PlainLet = self.let_source {
-            self.check_binding_is_irrefutable(pat, "local binding", Some(span))
+            self.check_binding_is_irrefutable(pat, "local binding", scrut, Some(span))
         } else {
-            let Ok(refutability) = self.is_let_irrefutable(pat) else { return };
+            let Ok(refutability) = self.is_let_irrefutable(pat, scrut) else { return };
             if matches!(refutability, Irrefutable) {
                 report_irrefutable_let_patterns(
                     self.tcx,
@@ -336,7 +416,7 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         expr_span: Span,
     ) {
         let scrut = &self.thir[scrut];
-        let cx = self.new_cx(Refutable, Some(expr_span), scrut.span);
+        let cx = self.new_cx(Refutable, Some(expr_span), Some(scrut), scrut.span);
 
         let mut tarms = Vec::with_capacity(arms.len());
         for &arm in arms {
@@ -377,7 +457,12 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
                 debug_assert_eq!(pat.span.desugaring_kind(), Some(DesugaringKind::ForLoop));
                 let PatKind::Variant { ref subpatterns, .. } = pat.kind else { bug!() };
                 let [pat_field] = &subpatterns[..] else { bug!() };
-                self.check_binding_is_irrefutable(&pat_field.pattern, "`for` loop binding", None);
+                self.check_binding_is_irrefutable(
+                    &pat_field.pattern,
+                    "`for` loop binding",
+                    None,
+                    None,
+                );
             } else {
                 self.error = Err(report_non_exhaustive_match(
                     &cx, self.thir, scrut_ty, scrut.span, witnesses, arms, expr_span,
@@ -457,16 +542,21 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         &mut self,
         pat: &Pat<'tcx>,
         refutability: RefutableFlag,
+        scrut: Option<&Expr<'tcx>>,
     ) -> Result<(MatchCheckCtxt<'p, 'tcx>, UsefulnessReport<'p, 'tcx>), ErrorGuaranteed> {
-        let cx = self.new_cx(refutability, None, pat.span);
+        let cx = self.new_cx(refutability, None, scrut, pat.span);
         let pat = self.lower_pattern(&cx, pat)?;
         let arms = [MatchArm { pat, hir_id: self.lint_level, has_guard: false }];
         let report = compute_match_usefulness(&cx, &arms, pat.ty());
         Ok((cx, report))
     }
 
-    fn is_let_irrefutable(&mut self, pat: &Pat<'tcx>) -> Result<RefutableFlag, ErrorGuaranteed> {
-        let (cx, report) = self.analyze_binding(pat, Refutable)?;
+    fn is_let_irrefutable(
+        &mut self,
+        pat: &Pat<'tcx>,
+        scrut: Option<&Expr<'tcx>>,
+    ) -> Result<RefutableFlag, ErrorGuaranteed> {
+        let (cx, report) = self.analyze_binding(pat, Refutable, scrut)?;
         // Report if the pattern is unreachable, which can only occur when the type is uninhabited.
         // This also reports unreachable sub-patterns.
         report_arm_reachability(&cx, &report);
@@ -476,10 +566,16 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
     }
 
     #[instrument(level = "trace", skip(self))]
-    fn check_binding_is_irrefutable(&mut self, pat: &Pat<'tcx>, origin: &str, sp: Option<Span>) {
+    fn check_binding_is_irrefutable(
+        &mut self,
+        pat: &Pat<'tcx>,
+        origin: &str,
+        scrut: Option<&Expr<'tcx>>,
+        sp: Option<Span>,
+    ) {
         let pattern_ty = pat.ty;
 
-        let Ok((cx, report)) = self.analyze_binding(pat, Irrefutable) else { return };
+        let Ok((cx, report)) = self.analyze_binding(pat, Irrefutable, scrut) else { return };
         let witnesses = report.non_exhaustiveness_witnesses;
         if witnesses.is_empty() {
             // The pattern is irrefutable.
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index 38a66522ac6..d86147b4699 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -551,6 +551,7 @@
 //! I (Nadrieril) prefer to put new tests in `ui/pattern/usefulness` unless there's a specific
 //! reason not to, for example if they crucially depend on a particular feature like `or_patterns`.
 
+use self::ValidityConstraint::*;
 use super::deconstruct_pat::{
     Constructor, ConstructorSet, DeconstructedPat, IntRange, MaybeInfiniteInt, SplitConstructorSet,
     WitnessPat,
@@ -587,11 +588,14 @@ pub(crate) struct MatchCheckCtxt<'p, 'tcx> {
     /// Lint level at the match.
     pub(crate) match_lint_level: HirId,
     /// The span of the whole match, if applicable.
-    pub(crate) match_span: Option<Span>,
+    pub(crate) whole_match_span: Option<Span>,
     /// Span of the scrutinee.
     pub(crate) scrut_span: Span,
     /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
     pub(crate) refutable: bool,
+    /// Whether the data at the scrutinee is known to be valid. This is false if the scrutinee comes
+    /// from a union field, a pointer deref, or a reference deref (pending opsem decisions).
+    pub(crate) known_valid_scrutinee: bool,
 }
 
 impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
@@ -620,12 +624,63 @@ pub(super) struct PatCtxt<'a, 'p, 'tcx> {
     pub(super) is_top_level: bool,
 }
 
+impl<'a, 'p, 'tcx> PatCtxt<'a, 'p, 'tcx> {
+    /// A `PatCtxt` when code other than `is_useful` needs one.
+    fn new_dummy(cx: &'a MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
+        PatCtxt { cx, ty, is_top_level: false }
+    }
+}
+
 impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("PatCtxt").field("ty", &self.ty).finish()
     }
 }
 
+/// In the matrix, tracks whether a given place (aka column) is known to contain a valid value or
+/// not.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(super) enum ValidityConstraint {
+    ValidOnly,
+    MaybeInvalid,
+}
+
+impl ValidityConstraint {
+    pub(super) fn from_bool(is_valid_only: bool) -> Self {
+        if is_valid_only { ValidOnly } else { MaybeInvalid }
+    }
+
+    /// If the place has validity given by `self` and we read that the value at the place has
+    /// constructor `ctor`, this computes what we can assume about the validity of the constructor
+    /// fields.
+    ///
+    /// Pending further opsem decisions, the current behavior is: validity is preserved, except
+    /// under `&` where validity is reset to `MaybeInvalid`.
+    pub(super) fn specialize<'tcx>(
+        self,
+        pcx: &PatCtxt<'_, '_, 'tcx>,
+        ctor: &Constructor<'tcx>,
+    ) -> Self {
+        // We preserve validity except when we go under a reference.
+        if matches!(ctor, Constructor::Single) && matches!(pcx.ty.kind(), ty::Ref(..)) {
+            // Validity of `x: &T` does not imply validity of `*x: T`.
+            MaybeInvalid
+        } else {
+            self
+        }
+    }
+}
+
+impl fmt::Display for ValidityConstraint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let s = match self {
+            ValidOnly => "✓",
+            MaybeInvalid => "?",
+        };
+        write!(f, "{s}")
+    }
+}
+
 /// Represents a pattern-tuple under investigation.
 #[derive(Clone)]
 struct PatStack<'p, 'tcx> {
@@ -770,10 +825,15 @@ impl<'p, 'tcx> fmt::Debug for MatrixRow<'p, 'tcx> {
 /// the matrix will correspond to `scrutinee.0.Some.0` and the second column to `scrutinee.1`.
 #[derive(Clone)]
 struct Matrix<'p, 'tcx> {
+    /// Vector of rows. The rows must form a rectangular 2D array. Moreover, all the patterns of
+    /// each column must have the same type. Each column corresponds to a place within the
+    /// scrutinee.
     rows: Vec<MatrixRow<'p, 'tcx>>,
     /// Stores an extra fictitious row full of wildcards. Mostly used to keep track of the type of
     /// each column. This must obey the same invariants as the real rows.
     wildcard_row: PatStack<'p, 'tcx>,
+    /// Track for each column/place whether it contains a known valid value.
+    place_validity: SmallVec<[ValidityConstraint; 2]>,
 }
 
 impl<'p, 'tcx> Matrix<'p, 'tcx> {
@@ -791,10 +851,22 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> {
     }
 
     /// Build a new matrix from an iterator of `MatchArm`s.
-    fn new(cx: &MatchCheckCtxt<'p, 'tcx>, arms: &[MatchArm<'p, 'tcx>], scrut_ty: Ty<'tcx>) -> Self {
+    fn new<'a>(
+        cx: &MatchCheckCtxt<'p, 'tcx>,
+        arms: &[MatchArm<'p, 'tcx>],
+        scrut_ty: Ty<'tcx>,
+        scrut_validity: ValidityConstraint,
+    ) -> Self
+    where
+        'p: 'a,
+    {
         let wild_pattern = cx.pattern_arena.alloc(DeconstructedPat::wildcard(scrut_ty, DUMMY_SP));
         let wildcard_row = PatStack::from_pattern(wild_pattern);
-        let mut matrix = Matrix { rows: Vec::with_capacity(arms.len()), wildcard_row };
+        let mut matrix = Matrix {
+            rows: Vec::with_capacity(arms.len()),
+            wildcard_row,
+            place_validity: smallvec![scrut_validity],
+        };
         for (row_id, arm) in arms.iter().enumerate() {
             let v = MatrixRow {
                 pats: PatStack::from_pattern(arm.pat),
@@ -858,7 +930,13 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> {
         ctor: &Constructor<'tcx>,
     ) -> Matrix<'p, 'tcx> {
         let wildcard_row = self.wildcard_row.pop_head_constructor(pcx, ctor);
-        let mut matrix = Matrix { rows: Vec::new(), wildcard_row };
+        let new_validity = self.place_validity[0].specialize(pcx, ctor);
+        let new_place_validity = std::iter::repeat(new_validity)
+            .take(ctor.arity(pcx))
+            .chain(self.place_validity[1..].iter().copied())
+            .collect();
+        let mut matrix =
+            Matrix { rows: Vec::new(), wildcard_row, place_validity: new_place_validity };
         for (i, row) in self.rows().enumerate() {
             if ctor.is_covered_by(pcx, row.head().ctor()) {
                 let new_row = row.pop_head_constructor(pcx, ctor, i);
@@ -877,27 +955,38 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> {
 /// + true  + [Second(true)]    +
 /// + false + [_]               +
 /// + _     + [_, _, tail @ ..] +
+/// | ✓     | ?                 | // column validity
 /// ```
 impl<'p, 'tcx> fmt::Debug for Matrix<'p, 'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "\n")?;
 
-        let Matrix { rows, .. } = self;
-        let pretty_printed_matrix: Vec<Vec<String>> =
-            rows.iter().map(|row| row.iter().map(|pat| format!("{pat:?}")).collect()).collect();
+        let mut pretty_printed_matrix: Vec<Vec<String>> = self
+            .rows
+            .iter()
+            .map(|row| row.iter().map(|pat| format!("{pat:?}")).collect())
+            .collect();
+        pretty_printed_matrix
+            .push(self.place_validity.iter().map(|validity| format!("{validity}")).collect());
 
-        let column_count = rows.iter().map(|row| row.len()).next().unwrap_or(0);
-        assert!(rows.iter().all(|row| row.len() == column_count));
+        let column_count = self.column_count();
+        assert!(self.rows.iter().all(|row| row.len() == column_count));
+        assert!(self.place_validity.len() == column_count);
         let column_widths: Vec<usize> = (0..column_count)
             .map(|col| pretty_printed_matrix.iter().map(|row| row[col].len()).max().unwrap_or(0))
             .collect();
 
-        for row in pretty_printed_matrix {
-            write!(f, "+")?;
+        for (row_i, row) in pretty_printed_matrix.into_iter().enumerate() {
+            let is_validity_row = row_i == self.rows.len();
+            let sep = if is_validity_row { "|" } else { "+" };
+            write!(f, "{sep}")?;
             for (column, pat_str) in row.into_iter().enumerate() {
                 write!(f, " ")?;
                 write!(f, "{:1$}", pat_str, column_widths[column])?;
-                write!(f, " +")?;
+                write!(f, " {sep}")?;
+            }
+            if is_validity_row {
+                write!(f, " // column validity")?;
             }
             write!(f, "\n")?;
         }
@@ -1287,7 +1376,7 @@ fn collect_nonexhaustive_missing_variants<'p, 'tcx>(
     let Some(ty) = column.head_ty() else {
         return Vec::new();
     };
-    let pcx = &PatCtxt { cx, ty, is_top_level: false };
+    let pcx = &PatCtxt::new_dummy(cx, ty);
 
     let set = column.analyze_ctors(pcx);
     if set.present.is_empty() {
@@ -1336,7 +1425,7 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
     let Some(ty) = column.head_ty() else {
         return;
     };
-    let pcx = &PatCtxt { cx, ty, is_top_level: false };
+    let pcx = &PatCtxt::new_dummy(cx, ty);
 
     let set = column.analyze_ctors(pcx);
 
@@ -1439,7 +1528,8 @@ pub(crate) fn compute_match_usefulness<'p, 'tcx>(
     arms: &[MatchArm<'p, 'tcx>],
     scrut_ty: Ty<'tcx>,
 ) -> UsefulnessReport<'p, 'tcx> {
-    let mut matrix = Matrix::new(cx, arms, scrut_ty);
+    let scrut_validity = ValidityConstraint::from_bool(cx.known_valid_scrutinee);
+    let mut matrix = Matrix::new(cx, arms, scrut_ty, scrut_validity);
     let non_exhaustiveness_witnesses = compute_exhaustiveness_and_usefulness(cx, &mut matrix, true);
 
     let non_exhaustiveness_witnesses: Vec<_> = non_exhaustiveness_witnesses.single_column();
@@ -1496,7 +1586,7 @@ pub(crate) fn compute_match_usefulness<'p, 'tcx>(
                 if !matches!(lint_level, rustc_session::lint::Level::Allow) {
                     let decorator = NonExhaustiveOmittedPatternLintOnArm {
                         lint_span: lint_level_source.span(),
-                        suggest_lint_on_match: cx.match_span.map(|span| span.shrink_to_lo()),
+                        suggest_lint_on_match: cx.whole_match_span.map(|span| span.shrink_to_lo()),
                         lint_level: lint_level.as_str(),
                         lint_name: "non_exhaustive_omitted_patterns",
                     };
-- 
cgit 1.4.1-3-g733a5


From ddef5b61f134834c81f8802d07ce3774e46143f6 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 18 Nov 2023 21:39:57 +0100
Subject: Don't warn an empty pattern unreachable if we're not sure the data is
 valid

---
 .../src/thir/pattern/deconstruct_pat.rs            |  44 ++-
 .../rustc_mir_build/src/thir/pattern/usefulness.rs |  85 +++--
 .../empty-types.exhaustive_patterns.stderr         | 389 +--------------------
 .../pattern/usefulness/empty-types.normal.stderr   | 130 +------
 tests/ui/pattern/usefulness/empty-types.rs         | 128 +++----
 tests/ui/pattern/usefulness/slice_of_empty.rs      |   6 +-
 tests/ui/pattern/usefulness/slice_of_empty.stderr  |  31 +-
 tests/ui/uninhabited/uninhabited-patterns.rs       |   4 +-
 tests/ui/uninhabited/uninhabited-patterns.stderr   |  20 +-
 9 files changed, 183 insertions(+), 654 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index f1ad0f25cd2..e66402a7eb3 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -787,9 +787,6 @@ pub(super) enum Constructor<'tcx> {
 }
 
 impl<'tcx> Constructor<'tcx> {
-    pub(super) fn is_wildcard(&self) -> bool {
-        matches!(self, Wildcard)
-    }
     pub(super) fn is_non_exhaustive(&self) -> bool {
         matches!(self, NonExhaustive)
     }
@@ -973,15 +970,17 @@ pub(super) enum ConstructorSet {
 /// constructors that exist in the type but are not present in the column.
 ///
 /// More formally, if we discard wildcards from the column, this respects the following constraints:
-/// 1. the union of `present` and `missing` covers the whole type
+/// 1. the union of `present`, `missing` and `missing_empty` covers all the constructors of the type
 /// 2. each constructor in `present` is covered by something in the column
-/// 3. no constructor in `missing` is covered by anything in the column
+/// 3. no constructor in `missing` or `missing_empty` is covered by anything in the column
 /// 4. each constructor in the column is equal to the union of one or more constructors in `present`
 /// 5. `missing` does not contain empty constructors (see discussion about emptiness at the top of
 ///    the file);
-/// 6. constructors in `present` and `missing` are split for the column; in other words, they are
-///    either fully included in or fully disjoint from each constructor in the column. In other
-///    words, there are no non-trivial intersections like between `0..10` and `5..15`.
+/// 6. `missing_empty` contains only empty constructors
+/// 7. constructors in `present`, `missing` and `missing_empty` are split for the column; in other
+///    words, they are either fully included in or fully disjoint from each constructor in the
+///    column. In yet other words, there are no non-trivial intersections like between `0..10` and
+///    `5..15`.
 ///
 /// We must be particularly careful with weird constructors like `Opaque`: they're not formally part
 /// of the `ConstructorSet` for the type, yet if we forgot to include them in `present` we would be
@@ -990,6 +989,7 @@ pub(super) enum ConstructorSet {
 pub(super) struct SplitConstructorSet<'tcx> {
     pub(super) present: SmallVec<[Constructor<'tcx>; 1]>,
     pub(super) missing: Vec<Constructor<'tcx>>,
+    pub(super) missing_empty: Vec<Constructor<'tcx>>,
 }
 
 impl ConstructorSet {
@@ -1132,10 +1132,10 @@ impl ConstructorSet {
         // Constructors in `ctors`, except wildcards and opaques.
         let mut seen = Vec::new();
         for ctor in ctors.cloned() {
-            if let Constructor::Opaque(..) = ctor {
-                present.push(ctor);
-            } else if !ctor.is_wildcard() {
-                seen.push(ctor);
+            match ctor {
+                Opaque(..) => present.push(ctor),
+                Wildcard => {} // discard wildcards
+                _ => seen.push(ctor),
             }
         }
 
@@ -1239,16 +1239,24 @@ impl ConstructorSet {
                 missing.push(NonExhaustive);
             }
             ConstructorSet::NoConstructors => {
-                if !pcx.is_top_level {
-                    missing_empty.push(NonExhaustive);
-                }
+                // In a `MaybeInvalid` place even an empty pattern may be reachable. We therefore
+                // add a dummy empty constructor here, which will be ignored if the place is
+                // `ValidOnly`.
+                missing_empty.push(NonExhaustive);
             }
         }
 
-        if !pcx.cx.tcx.features().exhaustive_patterns {
-            missing.extend(missing_empty);
+        // We have now grouped all the constructors into 3 buckets: present, missing, missing_empty.
+        // In the absence of the `exhaustive_patterns` feature however, we don't count nested empty
+        // types as empty. Only non-nested `!` or `enum Foo {}` are considered empty.
+        if !pcx.cx.tcx.features().exhaustive_patterns
+            && !(pcx.is_top_level && matches!(self, Self::NoConstructors))
+        {
+            // Treat all missing constructors as nonempty.
+            missing.extend(missing_empty.drain(..));
         }
-        SplitConstructorSet { present, missing }
+
+        SplitConstructorSet { present, missing, missing_empty }
     }
 }
 
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index d86147b4699..637cc38be2c 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -637,12 +637,18 @@ impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
     }
 }
 
-/// In the matrix, tracks whether a given place (aka column) is known to contain a valid value or
-/// not.
+/// Serves two purposes:
+/// - in a wildcard, tracks whether the wildcard matches only valid values (i.e. is a binding `_a`)
+///     or also invalid values (i.e. is a true `_` pattern).
+/// - in the matrix, track whether a given place (aka column) is known to contain a valid value or
+///     not.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub(super) enum ValidityConstraint {
     ValidOnly,
     MaybeInvalid,
+    /// Option for backwards compatibility: the place is not known to be valid but we allow omitting
+    /// `useful && !reachable` arms anyway.
+    MaybeInvalidButAllowOmittingArms,
 }
 
 impl ValidityConstraint {
@@ -650,19 +656,37 @@ impl ValidityConstraint {
         if is_valid_only { ValidOnly } else { MaybeInvalid }
     }
 
+    fn allow_omitting_side_effecting_arms(self) -> Self {
+        match self {
+            MaybeInvalid | MaybeInvalidButAllowOmittingArms => MaybeInvalidButAllowOmittingArms,
+            // There are no side-effecting empty arms here, nothing to do.
+            ValidOnly => ValidOnly,
+        }
+    }
+
+    pub(super) fn is_known_valid(self) -> bool {
+        matches!(self, ValidOnly)
+    }
+    pub(super) fn allows_omitting_empty_arms(self) -> bool {
+        matches!(self, ValidOnly | MaybeInvalidButAllowOmittingArms)
+    }
+
     /// If the place has validity given by `self` and we read that the value at the place has
     /// constructor `ctor`, this computes what we can assume about the validity of the constructor
     /// fields.
     ///
     /// Pending further opsem decisions, the current behavior is: validity is preserved, except
-    /// under `&` where validity is reset to `MaybeInvalid`.
+    /// inside `&` and union fields where validity is reset to `MaybeInvalid`.
     pub(super) fn specialize<'tcx>(
         self,
         pcx: &PatCtxt<'_, '_, 'tcx>,
         ctor: &Constructor<'tcx>,
     ) -> Self {
-        // We preserve validity except when we go under a reference.
-        if matches!(ctor, Constructor::Single) && matches!(pcx.ty.kind(), ty::Ref(..)) {
+        // We preserve validity except when we go inside a reference or a union field.
+        if matches!(ctor, Constructor::Single)
+            && (matches!(pcx.ty.kind(), ty::Ref(..))
+                || matches!(pcx.ty.kind(), ty::Adt(def, ..) if def.is_union()))
+        {
             // Validity of `x: &T` does not imply validity of `*x: T`.
             MaybeInvalid
         } else {
@@ -675,7 +699,7 @@ impl fmt::Display for ValidityConstraint {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let s = match self {
             ValidOnly => "✓",
-            MaybeInvalid => "?",
+            MaybeInvalid | MaybeInvalidButAllowOmittingArms => "?",
         };
         write!(f, "{s}")
     }
@@ -1202,9 +1226,9 @@ fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
         for row in matrix.rows_mut() {
             // All rows are useful until they're not.
             row.useful = true;
+            // When there's an unguarded row, the match is exhaustive and any subsequent row is not
+            // useful.
             if !row.is_under_guard {
-                // There's an unguarded row, so the match is exhaustive, and any subsequent row is
-                // unreachable.
                 return WitnessMatrix::empty();
             }
         }
@@ -1215,26 +1239,37 @@ fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
     debug!("ty: {ty:?}");
     let pcx = &PatCtxt { cx, ty, is_top_level };
 
+    // Whether the place/column we are inspecting is known to contain valid data.
+    let place_validity = matrix.place_validity[0];
+    // For backwards compability we allow omitting some empty arms that we ideally shouldn't.
+    let place_validity = place_validity.allow_omitting_side_effecting_arms();
+
     // Analyze the constructors present in this column.
     let ctors = matrix.heads().map(|p| p.ctor());
-    let split_set = ConstructorSet::for_ty(pcx.cx, pcx.ty).split(pcx, ctors);
-
+    let split_set = ConstructorSet::for_ty(cx, ty).split(pcx, ctors);
     let all_missing = split_set.present.is_empty();
-    let always_report_all = is_top_level && !IntRange::is_integral(pcx.ty);
-    // Whether we should report "Enum::A and Enum::C are missing" or "_ is missing".
-    let report_individual_missing_ctors = always_report_all || !all_missing;
 
+    // Build the set of constructors we will specialize with. It must cover the whole type.
     let mut split_ctors = split_set.present;
-    let mut only_report_missing = false;
     if !split_set.missing.is_empty() {
         // We need to iterate over a full set of constructors, so we add `Missing` to represent the
         // missing ones. This is explained under "Constructor Splitting" at the top of this file.
         split_ctors.push(Constructor::Missing);
-        // For diagnostic purposes we choose to only report the constructors that are missing. Since
-        // `Missing` matches only the wildcard rows, it matches fewer rows than any normal
-        // constructor and is therefore guaranteed to result in more witnesses. So skipping the
-        // other constructors does not jeopardize correctness.
-        only_report_missing = true;
+    } else if !split_set.missing_empty.is_empty() && !place_validity.is_known_valid() {
+        // The missing empty constructors are reachable if the place can contain invalid data.
+        split_ctors.push(Constructor::Missing);
+    }
+
+    // Decide what constructors to report.
+    let always_report_all = is_top_level && !IntRange::is_integral(pcx.ty);
+    // Whether we should report "Enum::A and Enum::C are missing" or "_ is missing".
+    let report_individual_missing_ctors = always_report_all || !all_missing;
+    // Which constructors are considered missing. We ensure that `!missing_ctors.is_empty() =>
+    // split_ctors.contains(Missing)`. The converse usually holds except in the
+    // `MaybeInvalidButAllowOmittingArms` backwards-compatibility case.
+    let mut missing_ctors = split_set.missing;
+    if !place_validity.allows_omitting_empty_arms() {
+        missing_ctors.extend(split_set.missing_empty);
     }
 
     let mut ret = WitnessMatrix::empty();
@@ -1246,11 +1281,19 @@ fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
             compute_exhaustiveness_and_usefulness(cx, &mut spec_matrix, false)
         });
 
-        if !only_report_missing || matches!(ctor, Constructor::Missing) {
+        let counts_for_exhaustiveness = match ctor {
+            Constructor::Missing => !missing_ctors.is_empty(),
+            // If there are missing constructors we'll report those instead. Since `Missing` matches
+            // only the wildcard rows, it matches fewer rows than this constructor, and is therefore
+            // guaranteed to result in the same or more witnesses. So skipping this does not
+            // jeopardize correctness.
+            _ => missing_ctors.is_empty(),
+        };
+        if counts_for_exhaustiveness {
             // Transform witnesses for `spec_matrix` into witnesses for `matrix`.
             witnesses.apply_constructor(
                 pcx,
-                &split_set.missing,
+                &missing_ctors,
                 &ctor,
                 report_individual_missing_ctors,
             );
diff --git a/tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr b/tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr
index 9a53b54704e..367aba3bdd6 100644
--- a/tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr
+++ b/tests/ui/pattern/usefulness/empty-types.exhaustive_patterns.stderr
@@ -31,12 +31,6 @@ LL +         _ => todo!(),
 LL +     }
    |
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:62:9
-   |
-LL |         &_ => {}
-   |         ^^
-
 error: unreachable pattern
   --> $DIR/empty-types.rs:69:9
    |
@@ -184,42 +178,6 @@ error: unreachable pattern
 LL |             _ => {}
    |             ^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:158:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:167:13
-   |
-LL |             Some(_) => {}
-   |             ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:171:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:175:13
-   |
-LL |             _a => {}
-   |             ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:180:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:185:13
-   |
-LL |             _ => {}
-   |             ^
-
 error: unreachable pattern
   --> $DIR/empty-types.rs:204:13
    |
@@ -250,48 +208,6 @@ error: unreachable pattern
 LL |             _ => {}
    |             ^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:234:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:239:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:245:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:251:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:256:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:262:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:268:13
-   |
-LL |             _ => {}
-   |             ^
-
 error: unreachable pattern
   --> $DIR/empty-types.rs:284:9
    |
@@ -316,18 +232,6 @@ error: unreachable pattern
 LL |         Err(_) => {}
    |         ^^^^^^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:296:9
-   |
-LL |         &_ => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:299:9
-   |
-LL |         Uninit { value: _ } => {}
-   |         ^^^^^^^^^^^^^^^^^^^
-
 error[E0004]: non-exhaustive patterns: type `&[!]` is non-empty
   --> $DIR/empty-types.rs:323:11
    |
@@ -342,24 +246,6 @@ LL +         _ => todo!(),
 LL +     }
    |
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:331:9
-   |
-LL |         [_] => {}
-   |         ^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:332:9
-   |
-LL |         [_, _, ..] => {}
-   |         ^^^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:337:9
-   |
-LL |         [_, _, _, ..] => {}
-   |         ^^^^^^^^^^^^^
-
 error[E0004]: non-exhaustive patterns: `&[]` not covered
   --> $DIR/empty-types.rs:334:11
    |
@@ -369,20 +255,9 @@ LL |     match slice_never {
    = note: the matched value is of type `&[!]`
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         [_, _, _, ..] => {}, &[] => todo!()
-   |                            ++++++++++++++++
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:341:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:345:9
+LL ~         [_, _, _, ..] => {},
+LL +         &[] => todo!()
    |
-LL |         _x => {}
-   |         ^^
 
 error[E0004]: non-exhaustive patterns: `&[]` not covered
   --> $DIR/empty-types.rs:347:11
@@ -430,18 +305,6 @@ error: unreachable pattern
 LL |         [_, ..] => {}
    |         ^^^^^^^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:375:9
-   |
-LL |         &[_, _, _] => {}
-   |         ^^^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:379:9
-   |
-LL |         &[_x, _, _] => {}
-   |         ^^^^^^^^^^^
-
 error[E0004]: non-exhaustive patterns: type `[!; 0]` is non-empty
   --> $DIR/empty-types.rs:383:11
    |
@@ -500,186 +363,6 @@ error: unreachable pattern
 LL |         _a => {}
    |         ^^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:436:9
-   |
-LL |         &_ => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:444:9
-   |
-LL |         &_a => {}
-   |         ^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:453:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:458:9
-   |
-LL |         _a => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:463:9
-   |
-LL |         &_ => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:468:9
-   |
-LL |         &_a => {}
-   |         ^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:475:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:479:9
-   |
-LL |         _a => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:485:9
-   |
-LL |         ref _a => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:494:9
-   |
-LL |         Some(_) => {}
-   |         ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:499:9
-   |
-LL |         Some(_a) => {}
-   |         ^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:504:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:509:9
-   |
-LL |         _a => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:514:14
-   |
-LL |         _a @ Some(_) => {}
-   |              ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:521:9
-   |
-LL |         ref _a => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:526:18
-   |
-LL |         ref _a @ Some(_) => {}
-   |                  ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:531:18
-   |
-LL |         ref _a @ Some(_b) => {}
-   |                  ^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:538:9
-   |
-LL |         Ok(_) => {}
-   |         ^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:542:9
-   |
-LL |         Ok(_) => {}
-   |         ^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:544:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:549:9
-   |
-LL |         Ok(_a) => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:553:9
-   |
-LL |         Ok(_a) => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:555:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:559:9
-   |
-LL |         Ok(_a) => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:561:9
-   |
-LL |         Err(_) => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:569:9
-   |
-LL |         (_, _) => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:573:9
-   |
-LL |         (_x, _) => {}
-   |         ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:577:9
-   |
-LL |         (_, _x) => {}
-   |         ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:581:9
-   |
-LL |         (0, _x) => {}
-   |         ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:583:9
-   |
-LL |         (1.., _) => {}
-   |         ^^^^^^^^
-
 error: unreachable pattern
   --> $DIR/empty-types.rs:598:9
    |
@@ -704,73 +387,7 @@ error: unreachable pattern
 LL |         _x if false => {}
    |         ^^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:613:9
-   |
-LL |         _ if false => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:615:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:622:9
-   |
-LL |         _a if false => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:624:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:629:9
-   |
-LL |         _a if false => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:634:9
-   |
-LL |         &_a if false => {}
-   |         ^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:641:9
-   |
-LL |         Ok(_x) if false => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:643:9
-   |
-LL |         Ok(_) => {}
-   |         ^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:645:9
-   |
-LL |         Err(_) => {}
-   |         ^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:650:9
-   |
-LL |         (_, _x) if false => {}
-   |         ^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:652:9
-   |
-LL |         (_, _) => {}
-   |         ^^^^^^
-
-error: aborting due to 113 previous errors
+error: aborting due to 49 previous errors
 
 Some errors have detailed explanations: E0004, E0005.
 For more information about an error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-types.normal.stderr b/tests/ui/pattern/usefulness/empty-types.normal.stderr
index b066393a61e..133a95a821b 100644
--- a/tests/ui/pattern/usefulness/empty-types.normal.stderr
+++ b/tests/ui/pattern/usefulness/empty-types.normal.stderr
@@ -234,12 +234,6 @@ LL ~             None => {},
 LL +             Some(_) => todo!()
    |
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:158:13
-   |
-LL |             _ => {}
-   |             ^
-
 error[E0004]: non-exhaustive patterns: `Some(_)` not covered
   --> $DIR/empty-types.rs:161:15
    |
@@ -258,18 +252,6 @@ LL ~             None => {},
 LL +             Some(_) => todo!()
    |
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:180:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:185:13
-   |
-LL |             _ => {}
-   |             ^
-
 error: unreachable pattern
   --> $DIR/empty-types.rs:204:13
    |
@@ -300,48 +282,6 @@ error: unreachable pattern
 LL |             _ => {}
    |             ^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:234:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:239:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:245:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:251:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:256:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:262:13
-   |
-LL |             _ => {}
-   |             ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:268:13
-   |
-LL |             _ => {}
-   |             ^
-
 error: unreachable pattern
   --> $DIR/empty-types.rs:284:9
    |
@@ -448,8 +388,9 @@ LL |     match slice_never {
    = note: the matched value is of type `&[!]`
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
-LL |         [_, _, _, ..] => {}, &[] | &[_] | &[_, _] => todo!()
-   |                            +++++++++++++++++++++++++++++++++
+LL ~         [_, _, _, ..] => {},
+LL +         &[] | &[_] | &[_, _] => todo!()
+   |
 
 error[E0004]: non-exhaustive patterns: `&[]` and `&[_, ..]` not covered
   --> $DIR/empty-types.rs:347:11
@@ -545,24 +486,6 @@ LL ~         &None => {},
 LL +         &Some(_) => todo!()
    |
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:475:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:479:9
-   |
-LL |         _a => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:485:9
-   |
-LL |         ref _a => {}
-   |         ^^^^^^
-
 error[E0004]: non-exhaustive patterns: `Some(_)` not covered
   --> $DIR/empty-types.rs:487:11
    |
@@ -595,8 +518,9 @@ note: `Result<!, !>` defined here
    = note: the matched value is of type `Result<!, !>`
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         Ok(_) => {}, Err(_) => todo!()
-   |                    +++++++++++++++++++
+LL ~         Ok(_) => {},
+LL +         Err(_) => todo!()
+   |
 
 error[E0004]: non-exhaustive patterns: `Err(_)` not covered
   --> $DIR/empty-types.rs:546:11
@@ -612,8 +536,9 @@ note: `Result<!, !>` defined here
    = note: the matched value is of type `Result<!, !>`
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         Ok(_a) => {}, Err(_) => todo!()
-   |                     +++++++++++++++++++
+LL ~         Ok(_a) => {},
+LL +         Err(_) => todo!()
+   |
 
 error[E0004]: non-exhaustive patterns: type `(u32, !)` is non-empty
   --> $DIR/empty-types.rs:565:11
@@ -653,36 +578,6 @@ error: unreachable pattern
 LL |         _x if false => {}
    |         ^^
 
-error: unreachable pattern
-  --> $DIR/empty-types.rs:613:9
-   |
-LL |         _ if false => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:615:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:622:9
-   |
-LL |         _a if false => {}
-   |         ^^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:624:9
-   |
-LL |         _ => {}
-   |         ^
-
-error: unreachable pattern
-  --> $DIR/empty-types.rs:629:9
-   |
-LL |         _a if false => {}
-   |         ^^
-
 error[E0004]: non-exhaustive patterns: `&_` not covered
   --> $DIR/empty-types.rs:631:11
    |
@@ -694,8 +589,9 @@ LL |     match ref_never {
    = note: match arms with guards don't count towards exhaustivity
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         &_a if false => {}, &_ => todo!()
-   |                           +++++++++++++++
+LL ~         &_a if false => {},
+LL +         &_ => todo!()
+   |
 
 error[E0004]: non-exhaustive patterns: `Some(_)` not covered
   --> $DIR/empty-types.rs:659:11
@@ -715,7 +611,7 @@ LL ~         None => {},
 LL +         Some(_) => todo!()
    |
 
-error: aborting due to 66 previous errors
+error: aborting due to 48 previous errors
 
 Some errors have detailed explanations: E0004, E0005.
 For more information about an error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/empty-types.rs b/tests/ui/pattern/usefulness/empty-types.rs
index fc422298a31..1e1d23e446d 100644
--- a/tests/ui/pattern/usefulness/empty-types.rs
+++ b/tests/ui/pattern/usefulness/empty-types.rs
@@ -59,7 +59,7 @@ fn basic(x: NeverBundle) {
     }
     match ref_never {
         // useful, reachable
-        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &_ => {}
     }
 
     let tuple_half_never: (u32, !) = x.tuple_half_never;
@@ -155,7 +155,7 @@ fn void_same_as_never(x: NeverBundle) {
         let ref_void: &Void = &x.void;
         match *ref_void {}
         match *ref_void {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         let ref_opt_void: &Option<Void> = &None;
         match *ref_opt_void {
@@ -164,25 +164,25 @@ fn void_same_as_never(x: NeverBundle) {
         }
         match *ref_opt_void {
             None => {}
-            Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+            Some(_) => {}
         }
         match *ref_opt_void {
             None => {}
-            _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+            _ => {}
         }
         match *ref_opt_void {
             None => {}
-            _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+            _a => {}
         }
         let union_void = Uninit::<Void>::new();
         match union_void.value {}
         match union_void.value {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         let ptr_void: *const Void = std::ptr::null();
         match *ptr_void {}
         match *ptr_void {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
     }
 }
@@ -231,41 +231,41 @@ fn invalid_scrutinees(x: NeverBundle) {
         // A pointer may point to a place with an invalid value.
         match *ptr_never {}
         match *ptr_never {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         // A reference may point to a place with an invalid value.
         match *ref_never {}
         match *ref_never {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         // This field access is a dereference.
         let ref_x: &NeverBundle = &x;
         match ref_x.never {}
         match ref_x.never {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         // This nested field access is a dereference.
         let nested_ref_x: &NestedNeverBundle = &nested_x;
         match nested_ref_x.0.never {}
         match nested_ref_x.0.never {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         // A cast does not load.
         match (*ptr_never as Void) {}
         match (*ptr_never as Void) {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         // A union field may contain invalid data.
         let union_never = Uninit::<!>::new();
         match union_never.value {}
         match union_never.value {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
         // Indexing is like a field access. This one accesses behind a reference.
         let slice_never: &[!] = &[];
         match slice_never[0] {}
         match slice_never[0] {
-            _ => {} //~ ERROR unreachable pattern
+            _ => {}
         }
     }
 }
@@ -293,10 +293,10 @@ fn nested_validity_tracking(bundle: NeverBundle) {
 
     // These should be considered !known_valid and not warn unreachable.
     match ref_never {
-        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &_ => {}
     }
     match union_never {
-        Uninit { value: _ } => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Uninit { value: _ } => {}
     }
 }
 
@@ -328,21 +328,21 @@ fn arrays_and_slices(x: NeverBundle) {
     }
     match slice_never {
         [] => {}
-        [_] => {}        //[exhaustive_patterns]~ ERROR unreachable pattern
-        [_, _, ..] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        [_] => {}
+        [_, _, ..] => {}
     }
     match slice_never {
         //[normal]~^ ERROR `&[]`, `&[_]` and `&[_, _]` not covered
         //[exhaustive_patterns]~^^ ERROR `&[]` not covered
-        [_, _, _, ..] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        [_, _, _, ..] => {}
     }
     match slice_never {
         [] => {}
-        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _ => {}
     }
     match slice_never {
         [] => {}
-        _x => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _x => {}
     }
     match slice_never {
         //[normal]~^ ERROR `&[]` and `&[_, ..]` not covered
@@ -372,11 +372,11 @@ fn arrays_and_slices(x: NeverBundle) {
     let ref_array_3_never: &[!; 3] = &array_3_never;
     match ref_array_3_never {
         // useful, reachable
-        &[_, _, _] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &[_, _, _] => {}
     }
     match ref_array_3_never {
         // useful, !reachable
-        &[_x, _, _] => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &[_x, _, _] => {}
     }
 
     let array_0_never: [!; 0] = [];
@@ -433,7 +433,7 @@ fn bindings(x: NeverBundle) {
     }
     match ref_never {
         // useful, reachable
-        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &_ => {}
     }
     match ref_never {
         // useful, reachable
@@ -441,7 +441,7 @@ fn bindings(x: NeverBundle) {
     }
     match ref_never {
         // useful, !reachable
-        &_a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &_a => {}
     }
     match ref_opt_never {
         //[normal]~^ ERROR non-exhaustive
@@ -450,39 +450,39 @@ fn bindings(x: NeverBundle) {
     match ref_opt_never {
         &None => {}
         // useful, reachable
-        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _ => {}
     }
     match ref_opt_never {
         &None => {}
         // useful, reachable
-        _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _a => {}
     }
     match ref_opt_never {
         &None => {}
         // useful, reachable
-        &_ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &_ => {}
     }
     match ref_opt_never {
         &None => {}
         // useful, !reachable
-        &_a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &_a => {}
     }
 
     // On a !known_valid place.
     match *ref_never {}
     match *ref_never {
         // useful, reachable
-        _ => {} //~ ERROR unreachable pattern
+        _ => {}
     }
     match *ref_never {
         // useful, !reachable
-        _a => {} //~ ERROR unreachable pattern
+        _a => {}
     }
     // This is equivalent to `match ref_never { _a => {} }`. In other words, it asserts validity of
     // `ref_never` but says nothing of the data at `*ref_never`.
     match *ref_never {
         // useful, reachable
-        ref _a => {} //~ ERROR unreachable pattern
+        ref _a => {}
     }
     match *ref_opt_never {
         //[normal]~^ ERROR non-exhaustive
@@ -491,74 +491,74 @@ fn bindings(x: NeverBundle) {
     match *ref_opt_never {
         None => {}
         // useful, reachable
-        Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Some(_) => {}
     }
     match *ref_opt_never {
         None => {}
         // useful, !reachable
-        Some(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Some(_a) => {}
     }
     match *ref_opt_never {
         None => {}
         // useful, reachable
-        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _ => {}
     }
     match *ref_opt_never {
         None => {}
         // useful, !reachable
-        _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _a => {}
     }
     match *ref_opt_never {
         None => {}
         // useful, !reachable
-        _a @ Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _a @ Some(_) => {}
     }
     // This is equivalent to `match ref_opt_never { None => {}, _a => {} }`. In other words, it
     // asserts validity of `ref_opt_never` but says nothing of the data at `*ref_opt_never`.
     match *ref_opt_never {
         None => {}
         // useful, reachable
-        ref _a => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        ref _a => {}
     }
     match *ref_opt_never {
         None => {}
         // useful, reachable
-        ref _a @ Some(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        ref _a @ Some(_) => {}
     }
     match *ref_opt_never {
         None => {}
         // useful, !reachable
-        ref _a @ Some(_b) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        ref _a @ Some(_b) => {}
     }
 
     let ref_res_never: &Result<!, !> = &x.result_never;
     match *ref_res_never {
         //[normal]~^ ERROR non-exhaustive
         // useful, reachable
-        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Ok(_) => {}
     }
     match *ref_res_never {
         // useful, reachable
-        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Ok(_) => {}
         // useful, reachable
-        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _ => {}
     }
     match *ref_res_never {
         //[normal]~^ ERROR non-exhaustive
         // useful, !reachable
-        Ok(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Ok(_a) => {}
     }
     match *ref_res_never {
         // useful, !reachable
-        Ok(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Ok(_a) => {}
         // useful, reachable
-        _ => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        _ => {}
     }
     match *ref_res_never {
         // useful, !reachable
-        Ok(_a) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Ok(_a) => {}
         // useful, reachable
-        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Err(_) => {}
     }
 
     let ref_tuple_half_never: &(u32, !) = &x.tuple_half_never;
@@ -566,21 +566,21 @@ fn bindings(x: NeverBundle) {
     //[normal]~^ ERROR non-empty
     match *ref_tuple_half_never {
         // useful, reachable
-        (_, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        (_, _) => {}
     }
     match *ref_tuple_half_never {
         // useful, reachable
-        (_x, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        (_x, _) => {}
     }
     match *ref_tuple_half_never {
         // useful, !reachable
-        (_, _x) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        (_, _x) => {}
     }
     match *ref_tuple_half_never {
         // useful, !reachable
-        (0, _x) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        (0, _x) => {}
         // useful, reachable
-        (1.., _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        (1.., _) => {}
     }
 }
 
@@ -610,46 +610,46 @@ fn guards_and_validity(x: NeverBundle) {
     // If the pattern under the guard doesn't load, all is normal.
     match *ref_never {
         // useful, reachable
-        _ if false => {} //~ ERROR unreachable pattern
+        _ if false => {}
         // useful, reachable
-        _ => {} //~ ERROR unreachable pattern
+        _ => {}
     }
     // Now the madness commences. The guard caused a load of the value thus asserting validity. So
     // there's no invalid value for `_` to catch. So the second pattern is unreachable despite the
     // guard not being taken.
     match *ref_never {
         // useful, !reachable
-        _a if false => {} //~ ERROR unreachable pattern
+        _a if false => {}
         // !useful, !reachable
-        _ => {} //~ ERROR unreachable pattern
+        _ => {}
     }
     // The above still applies to the implicit `_` pattern used for exhaustiveness.
     match *ref_never {
         // useful, !reachable
-        _a if false => {} //~ ERROR unreachable pattern
+        _a if false => {}
     }
     match ref_never {
         //[normal]~^ ERROR non-exhaustive
         // useful, !reachable
-        &_a if false => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        &_a if false => {}
     }
 
     // Same but with subpatterns.
     let ref_result_never: &Result<!, !> = &x.result_never;
     match *ref_result_never {
         // useful, !reachable
-        Ok(_x) if false => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Ok(_x) if false => {}
         // !useful, !reachable
-        Ok(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Ok(_) => {}
         // useful, !reachable
-        Err(_) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        Err(_) => {}
     }
     let ref_tuple_never: &(!, !) = &x.tuple_never;
     match *ref_tuple_never {
         // useful, !reachable
-        (_, _x) if false => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        (_, _x) if false => {}
         // !useful, !reachable
-        (_, _) => {} //[exhaustive_patterns]~ ERROR unreachable pattern
+        (_, _) => {}
     }
 }
 
diff --git a/tests/ui/pattern/usefulness/slice_of_empty.rs b/tests/ui/pattern/usefulness/slice_of_empty.rs
index fe068871195..3cbd0eba57f 100644
--- a/tests/ui/pattern/usefulness/slice_of_empty.rs
+++ b/tests/ui/pattern/usefulness/slice_of_empty.rs
@@ -11,12 +11,12 @@ fn foo(nevers: &[!]) {
 
     match nevers {
         &[] => (),
-        &[_] => (),        //~ ERROR unreachable pattern
-        &[_, _, ..] => (), //~ ERROR unreachable pattern
+        &[_] => (),
+        &[_, _, ..] => (),
     };
 
     match nevers {
         //~^ ERROR non-exhaustive patterns: `&[]` not covered
-        &[_] => (), //~ ERROR unreachable pattern
+        &[_] => (),
     };
 }
diff --git a/tests/ui/pattern/usefulness/slice_of_empty.stderr b/tests/ui/pattern/usefulness/slice_of_empty.stderr
index 07bb6b3a67d..d56360d4cec 100644
--- a/tests/ui/pattern/usefulness/slice_of_empty.stderr
+++ b/tests/ui/pattern/usefulness/slice_of_empty.stderr
@@ -1,27 +1,3 @@
-error: unreachable pattern
-  --> $DIR/slice_of_empty.rs:14:9
-   |
-LL |         &[_] => (),
-   |         ^^^^
-   |
-note: the lint level is defined here
-  --> $DIR/slice_of_empty.rs:3:9
-   |
-LL | #![deny(unreachable_patterns)]
-   |         ^^^^^^^^^^^^^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/slice_of_empty.rs:15:9
-   |
-LL |         &[_, _, ..] => (),
-   |         ^^^^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/slice_of_empty.rs:20:9
-   |
-LL |         &[_] => (),
-   |         ^^^^
-
 error[E0004]: non-exhaustive patterns: `&[]` not covered
   --> $DIR/slice_of_empty.rs:18:11
    |
@@ -31,9 +7,10 @@ LL |     match nevers {
    = note: the matched value is of type `&[!]`
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         &[_] => (), &[] => todo!(),
-   |                   ++++++++++++++++
+LL ~         &[_] => (),
+LL ~         &[] => todo!(),
+   |
 
-error: aborting due to 4 previous errors
+error: aborting due to 1 previous error
 
 For more information about this error, try `rustc --explain E0004`.
diff --git a/tests/ui/uninhabited/uninhabited-patterns.rs b/tests/ui/uninhabited/uninhabited-patterns.rs
index 43b19e790e2..4e90691e5c8 100644
--- a/tests/ui/uninhabited/uninhabited-patterns.rs
+++ b/tests/ui/uninhabited/uninhabited-patterns.rs
@@ -22,14 +22,14 @@ fn main() {
 
     match x {
         &[] => (),
-        &[..] => (), //~ ERROR unreachable pattern
+        &[..] => (),
     };
 
     let x: Result<Box<NotSoSecretlyEmpty>, &[Result<!, !>]> = Err(&[]);
     match x {
         Ok(box _) => (), //~ ERROR unreachable pattern
         Err(&[]) => (),
-        Err(&[..]) => (), //~ ERROR unreachable pattern
+        Err(&[..]) => (),
     }
 
     let x: Result<foo::SecretlyEmpty, Result<NotSoSecretlyEmpty, u32>> = Err(Err(123));
diff --git a/tests/ui/uninhabited/uninhabited-patterns.stderr b/tests/ui/uninhabited/uninhabited-patterns.stderr
index 19f34a52bdb..a6fda88f032 100644
--- a/tests/ui/uninhabited/uninhabited-patterns.stderr
+++ b/tests/ui/uninhabited/uninhabited-patterns.stderr
@@ -1,8 +1,8 @@
 error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:25:9
+  --> $DIR/uninhabited-patterns.rs:30:9
    |
-LL |         &[..] => (),
-   |         ^^^^^
+LL |         Ok(box _) => (),
+   |         ^^^^^^^^^
    |
 note: the lint level is defined here
   --> $DIR/uninhabited-patterns.rs:4:9
@@ -10,18 +10,6 @@ note: the lint level is defined here
 LL | #![deny(unreachable_patterns)]
    |         ^^^^^^^^^^^^^^^^^^^^
 
-error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:30:9
-   |
-LL |         Ok(box _) => (),
-   |         ^^^^^^^^^
-
-error: unreachable pattern
-  --> $DIR/uninhabited-patterns.rs:32:9
-   |
-LL |         Err(&[..]) => (),
-   |         ^^^^^^^^^^
-
 error: unreachable pattern
   --> $DIR/uninhabited-patterns.rs:39:9
    |
@@ -34,5 +22,5 @@ error: unreachable pattern
 LL |     while let Some(_y) = foo() {
    |               ^^^^^^^^
 
-error: aborting due to 5 previous errors
+error: aborting due to 3 previous errors
 
-- 
cgit 1.4.1-3-g733a5


From 3818fc02c4a55eb8cb12c90505bc88ae6cb983e8 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Sat, 9 Dec 2023 00:00:53 +0000
Subject: Lower constness span in host param

---
 compiler/rustc_ast_lowering/src/item.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 9d1f2684c39..ced267b8a36 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -1385,6 +1385,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let host_param_parts = if let Const::Yes(span) = constness
             && self.tcx.features().effects
         {
+            let span = self.lower_span(span);
             let param_node_id = self.next_node_id();
             let hir_id = self.next_id();
             let def_id = self.create_def(
-- 
cgit 1.4.1-3-g733a5


From 97de8fba56f762edea9c55f27207657aff065114 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Sat, 9 Dec 2023 00:02:34 +0000
Subject: Lower spans for opaque duplicated lifetimes, const infer vars

---
 compiler/rustc_ast_lowering/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 753650f7324..dd52c5def54 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -1678,7 +1678,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     duplicated_lifetime_node_id,
                     lifetime.ident.name,
                     DefKind::LifetimeParam,
-                    lifetime.ident.span,
+                    self.lower_span(lifetime.ident.span),
                 );
                 captured_to_synthesized_mapping.insert(old_def_id, duplicated_lifetime_def_id);
                 // FIXME: Instead of doing this, we could move this whole loop
@@ -1687,7 +1687,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 synthesized_lifetime_definitions.push((
                     duplicated_lifetime_node_id,
                     duplicated_lifetime_def_id,
-                    lifetime.ident,
+                    self.lower_ident(lifetime.ident),
                 ));
 
                 // Now make an arg that we can use for the generic params of the opaque tykind.
@@ -2252,7 +2252,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         match c.value.kind {
             ExprKind::Underscore => {
                 if self.tcx.features().generic_arg_infer {
-                    hir::ArrayLen::Infer(self.lower_node_id(c.id), c.value.span)
+                    hir::ArrayLen::Infer(self.lower_node_id(c.id), self.lower_span(c.value.span))
                 } else {
                     feature_err(
                         &self.tcx.sess.parse_sess,
-- 
cgit 1.4.1-3-g733a5


From c980fae35cf07ed73ff52ff099c21999c1e91a15 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Sat, 9 Dec 2023 00:10:29 +0000
Subject: Lower some forgotten coroutine spans

---
 compiler/rustc_ast_lowering/src/expr.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index c287c65ff36..99691f43f91 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -634,7 +634,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // Resume argument type: `ResumeTy`
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::Async,
-            span,
+            self.lower_span(span),
             Some(self.allow_gen_future.clone()),
         );
         let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
@@ -766,7 +766,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // Resume argument type: `ResumeTy`
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::Async,
-            span,
+            self.lower_span(span),
             Some(self.allow_gen_future.clone()),
         );
         let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
-- 
cgit 1.4.1-3-g733a5


From c3df51a976dfa69f0f1869997fdf79516ba6afab Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 9 Dec 2023 00:50:52 +0100
Subject: Some types cannot show up as the type of a pattern

---
 compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index e66402a7eb3..ef20b0f039b 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -1101,13 +1101,12 @@ impl ConstructorSet {
             | ty::Dynamic(_, _, _)
             | ty::Closure(_, _)
             | ty::Coroutine(_, _, _)
-            | ty::CoroutineWitness(_, _)
             | ty::Alias(_, _)
             | ty::Param(_)
-            | ty::Bound(_, _)
-            | ty::Placeholder(_)
-            | ty::Infer(_)
             | ty::Error(_) => Self::Unlistable,
+            ty::CoroutineWitness(_, _) | ty::Bound(_, _) | ty::Placeholder(_) | ty::Infer(_) => {
+                bug!("Encountered unexpected type in `ConstructorSet::for_ty`: {ty:?}")
+            }
         }
     }
 
-- 
cgit 1.4.1-3-g733a5


From cb6984217f12acd1da6eb7f244effe2c9f9f11f8 Mon Sep 17 00:00:00 2001
From: Young-Flash <871946895@qq.com>
Date: Sat, 9 Dec 2023 17:49:40 +0800
Subject: chore: add test case for type with generic

---
 .../suggest-assoc-fn-call-without-receiver.fixed   | 14 +++++++
 .../suggest-assoc-fn-call-without-receiver.rs      | 14 +++++++
 .../suggest-assoc-fn-call-without-receiver.stderr  | 44 ++++++++++++++++++++--
 3 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed
index 6e679134d63..61f06d802b6 100644
--- a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed
+++ b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.fixed
@@ -7,10 +7,24 @@ impl A {
     fn test(_a: Self, _b: i32) {}
 }
 
+struct B<T> {
+    _b: T
+}
+impl<T> B<T> {
+    fn hello(_a: i32) {}
+    fn test(_a: Self, _b: i32) {}
+}
+
 fn main() {
     let _a = A {};
     A::hello(1);
     //~^ ERROR no method named `hello` found
     A::test(_a, 1);
     //~^ ERROR no method named `test` found
+
+    let _b = B {_b: ""};
+    B::<&str>::hello(1);
+    //~^ ERROR no method named `hello` found
+    B::<&str>::test(_b, 1);
+    //~^ ERROR no method named `test` found
 }
diff --git a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs
index 67c2cc1bed5..07e614f0c15 100644
--- a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs
+++ b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.rs
@@ -7,10 +7,24 @@ impl A {
     fn test(_a: Self, _b: i32) {}
 }
 
+struct B<T> {
+    _b: T
+}
+impl<T> B<T> {
+    fn hello(_a: i32) {}
+    fn test(_a: Self, _b: i32) {}
+}
+
 fn main() {
     let _a = A {};
     _a.hello(1);
     //~^ ERROR no method named `hello` found
     _a.test(1);
     //~^ ERROR no method named `test` found
+
+    let _b = B {_b: ""};
+    _b.hello(1);
+    //~^ ERROR no method named `hello` found
+    _b.test(1);
+    //~^ ERROR no method named `test` found
 }
diff --git a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr
index ed227cbab39..793595784d9 100644
--- a/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr
+++ b/tests/ui/suggestions/suggest-assoc-fn-call-without-receiver.stderr
@@ -1,5 +1,5 @@
 error[E0599]: no method named `hello` found for struct `A` in the current scope
-  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:12:8
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:20:8
    |
 LL | struct A {}
    | -------- method `hello` not found for this struct
@@ -18,7 +18,7 @@ LL |     fn hello(_a: i32) {}
    |     ^^^^^^^^^^^^^^^^^
 
 error[E0599]: no method named `test` found for struct `A` in the current scope
-  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:14:8
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:22:8
    |
 LL | struct A {}
    | -------- method `test` not found for this struct
@@ -36,6 +36,44 @@ note: the candidate is defined in an impl for the type `A`
 LL |     fn test(_a: Self, _b: i32) {}
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error: aborting due to 2 previous errors
+error[E0599]: no method named `hello` found for struct `B<&str>` in the current scope
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:26:8
+   |
+LL | struct B<T> {
+   | ----------- method `hello` not found for this struct
+...
+LL |     _b.hello(1);
+   |     ---^^^^^---
+   |     |  |
+   |     |  this is an associated function, not a method
+   |     help: use associated function syntax instead: `B::<&str>::hello(1)`
+   |
+   = note: found the following associated functions; to be used as methods, functions must have a `self` parameter
+note: the candidate is defined in an impl for the type `B<T>`
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:14:5
+   |
+LL |     fn hello(_a: i32) {}
+   |     ^^^^^^^^^^^^^^^^^
+
+error[E0599]: no method named `test` found for struct `B<&str>` in the current scope
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:28:8
+   |
+LL | struct B<T> {
+   | ----------- method `test` not found for this struct
+...
+LL |     _b.test(1);
+   |     ---^^^^---
+   |     |  |
+   |     |  this is an associated function, not a method
+   |     help: use associated function syntax instead: `B::<&str>::test(_b, 1)`
+   |
+   = note: found the following associated functions; to be used as methods, functions must have a `self` parameter
+note: the candidate is defined in an impl for the type `B<T>`
+  --> $DIR/suggest-assoc-fn-call-without-receiver.rs:15:5
+   |
+LL |     fn test(_a: Self, _b: i32) {}
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 4 previous errors
 
 For more information about this error, try `rustc --explain E0599`.
-- 
cgit 1.4.1-3-g733a5


From ef796db5f0d4eecce0dd2ae9fff224542fb88f5b Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Sat, 9 Dec 2023 09:47:14 +0000
Subject: add test for inductive cycle hangs

---
 .../coinduction/fixpoint-exponential-growth.rs     |  8 ++++--
 .../coinduction/fixpoint-exponential-growth.stderr |  4 +--
 .../new-solver/cycles/inductive-fixpoint-hang.rs   | 33 ++++++++++++++++++++++
 .../cycles/inductive-fixpoint-hang.stderr          | 16 +++++++++++
 4 files changed, 57 insertions(+), 4 deletions(-)
 create mode 100644 tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.rs
 create mode 100644 tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.stderr

diff --git a/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.rs b/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.rs
index 44e763ef990..07c7d4fb29c 100644
--- a/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.rs
+++ b/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.rs
@@ -3,8 +3,12 @@
 // Proving `W<?0>: Trait` instantiates `?0` with `(W<?1>, W<?2>)` and then
 // proves `W<?1>: Trait` and `W<?2>: Trait`, resulting in a coinductive cycle.
 //
-// Proving coinductive cycles runs until we reach a fixpoint. This fixpoint is
-// never reached here and each step doubles the amount of nested obligations.
+// Proving coinductive cycles runs until we reach a fixpoint. However, after
+// computing `try_evaluate_added_goals` in the second fixpoint iteration, the
+// self type already has a depth equal to the number of steps. This results
+// in enormous constraints, causing the canonicalizer to hang without ever
+// reaching the recursion limit. We currently avoid that by erasing the constraints
+// from overflow.
 //
 // This previously caused a hang in the trait solver, see
 // https://github.com/rust-lang/trait-system-refactor-initiative/issues/13.
diff --git a/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.stderr b/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.stderr
index 05aaf6108f1..150100f2c53 100644
--- a/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.stderr
+++ b/tests/ui/traits/new-solver/cycles/coinduction/fixpoint-exponential-growth.stderr
@@ -1,12 +1,12 @@
 error[E0275]: overflow evaluating the requirement `W<_>: Trait`
-  --> $DIR/fixpoint-exponential-growth.rs:29:13
+  --> $DIR/fixpoint-exponential-growth.rs:33:13
    |
 LL |     impls::<W<_>>();
    |             ^^^^
    |
    = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`fixpoint_exponential_growth`)
 note: required by a bound in `impls`
-  --> $DIR/fixpoint-exponential-growth.rs:26:13
+  --> $DIR/fixpoint-exponential-growth.rs:30:13
    |
 LL | fn impls<T: Trait>() {}
    |             ^^^^^ required by this bound in `impls`
diff --git a/tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.rs b/tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.rs
new file mode 100644
index 00000000000..062c6ae98d5
--- /dev/null
+++ b/tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.rs
@@ -0,0 +1,33 @@
+// compile-flags: -Ztrait-solver=next
+
+// This currently hangs if we do not erase constraints from
+// overflow.
+//
+// We set the provisional result of `W<?0>` to `?0 := W<_>`.
+// The next iteration does not simply result in a `?0 := W<W<_>` constraint as
+// one might expect, but instead each time we evaluate the nested `W<T>` goal we
+// apply the previously returned constraints: the first fixpoint iteration goes
+// as follows: `W<?1>: Trait` constrains `?1` to `W<?2>`, we then evaluate
+// `W<W<?2>>: Trait` the next time we try to prove the nested goal. This results
+// inn `W<W<W<?3>>>` and so on. This goes on until we reach overflow in
+// `try_evaluate_added_goals`.  This means the provisional result after the
+// second fixpoint iteration is already `W<W<W<...>>>` with a size proportional
+// to the number of steps in `try_evaluate_added_goals`. The size then continues
+// to grow. The exponential blowup from having 2 nested goals per impl causes
+// the solver to hang without hitting the recursion limit.
+trait Trait {}
+
+struct W<T: ?Sized>(*const T);
+
+impl<T: ?Sized> Trait for W<W<T>>
+where
+    W<T>: Trait,
+    W<T>: Trait,
+{}
+
+fn impls_trait<T: Trait>() {}
+
+fn main() {
+    impls_trait::<W<_>>();
+    //~^ ERROR overflow evaluating the requirement
+}
diff --git a/tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.stderr b/tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.stderr
new file mode 100644
index 00000000000..42451920744
--- /dev/null
+++ b/tests/ui/traits/new-solver/cycles/inductive-fixpoint-hang.stderr
@@ -0,0 +1,16 @@
+error[E0275]: overflow evaluating the requirement `W<_>: Trait`
+  --> $DIR/inductive-fixpoint-hang.rs:31:19
+   |
+LL |     impls_trait::<W<_>>();
+   |                   ^^^^
+   |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`inductive_fixpoint_hang`)
+note: required by a bound in `impls_trait`
+  --> $DIR/inductive-fixpoint-hang.rs:28:19
+   |
+LL | fn impls_trait<T: Trait>() {}
+   |                   ^^^^^ required by this bound in `impls_trait`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0275`.
-- 
cgit 1.4.1-3-g733a5


From bba9862b951d852aa67e7b91718a09b1db062942 Mon Sep 17 00:00:00 2001
From: Urgau <urgau@numericable.fr>
Date: Thu, 7 Dec 2023 12:04:00 +0100
Subject: Strengthen well known check-cfg names and values test

---
 compiler/rustc_session/src/config.rs        |   3 +
 tests/ui/check-cfg/well-known-values.rs     | 107 ++++++++++---
 tests/ui/check-cfg/well-known-values.stderr | 240 ++++++++++++++++++++++++----
 3 files changed, 294 insertions(+), 56 deletions(-)

diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 4e16c378a58..ef465f371d2 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -1422,6 +1422,9 @@ impl CheckCfg {
         };
 
         // NOTE: This should be kept in sync with `default_configuration`
+        //
+        // When adding a new config here you should also update
+        // `tests/ui/check-cfg/well-known-values.rs`.
 
         let panic_values = &PanicStrategy::all();
 
diff --git a/tests/ui/check-cfg/well-known-values.rs b/tests/ui/check-cfg/well-known-values.rs
index 8b56c8729d8..39a470c202f 100644
--- a/tests/ui/check-cfg/well-known-values.rs
+++ b/tests/ui/check-cfg/well-known-values.rs
@@ -1,41 +1,104 @@
-// This test check that we lint on non well known values and that we don't lint on well known
-// values
+// This test check that we recognize all the well known config names
+// and that we correctly lint on unexpected values.
+//
+// This test also serve as an "anti-regression" for the well known
+// values since the suggestion shows them.
 //
 // check-pass
 // compile-flags: --check-cfg=cfg() -Z unstable-options
 
-#[cfg(target_os = "linuz")]
+#![feature(cfg_overflow_checks)]
+#![feature(cfg_relocation_model)]
+#![feature(cfg_sanitize)]
+#![feature(cfg_target_abi)]
+#![feature(cfg_target_has_atomic)]
+#![feature(cfg_target_has_atomic_equal_alignment)]
+#![feature(cfg_target_thread_local)]
+
+// This part makes sure that none of the well known names are
+// unexpected.
+//
+// BUT to make sure that no expected values changes without
+// being noticed we pass them a obviously wrong value so the
+// diagnostic prints the list of expected values.
+#[cfg(any(
+    // tidy-alphabetical-start
+    debug_assertions = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    doc = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    doctest = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    miri = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    overflow_checks = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    panic = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    proc_macro = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    relocation_model = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    sanitize = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_abi = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_arch = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_endian = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_env = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_family = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_feature = "_UNEXPECTED_VALUE", // currently *any* values are "expected"
+    target_has_atomic = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_has_atomic_equal_alignment = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_has_atomic_load_store = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_os = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_pointer_width = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_thread_local = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    target_vendor = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    test = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    unix = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    windows = "_UNEXPECTED_VALUE",
+    //~^ WARN unexpected `cfg` condition value
+    // tidy-alphabetical-end
+))]
+fn unexpected_values() {}
+
+#[cfg(target_os = "linuz")] // testing that we suggest `linux`
 //~^ WARNING unexpected `cfg` condition value
 fn target_os_linux_misspell() {}
 
+// The #[cfg]s below serve as a safeguard to make sure we
+// don't lint when using an expected well-known name and
+// value, only a small subset of all possible expected
+// configs are tested, since we already test the names
+// above and don't need to test all values, just different
+// combinations (without value, with value, both...).
+
 #[cfg(target_os = "linux")]
 fn target_os_linux() {}
 
-#[cfg(target_has_atomic = "0")]
-//~^ WARNING unexpected `cfg` condition value
-fn target_has_atomic_invalid() {}
-
 #[cfg(target_has_atomic = "8")]
-fn target_has_atomic() {}
+fn target_has_atomic_8() {}
 
-#[cfg(unix = "aa")]
-//~^ WARNING unexpected `cfg` condition value
-fn unix_with_value() {}
+#[cfg(target_has_atomic)]
+fn target_has_atomic() {}
 
 #[cfg(unix)]
 fn unix() {}
 
-#[cfg(miri = "miri")]
-//~^ WARNING unexpected `cfg` condition value
-fn miri_with_value() {}
-
-#[cfg(miri)]
-fn miri() {}
-
-#[cfg(doc = "linux")]
-//~^ WARNING unexpected `cfg` condition value
-fn doc_with_value() {}
-
 #[cfg(doc)]
 fn doc() {}
 
diff --git a/tests/ui/check-cfg/well-known-values.stderr b/tests/ui/check-cfg/well-known-values.stderr
index 6877d8f5bb7..a6b9c75a142 100644
--- a/tests/ui/check-cfg/well-known-values.stderr
+++ b/tests/ui/check-cfg/well-known-values.stderr
@@ -1,53 +1,225 @@
-warning: unexpected `cfg` condition value: `linuz`
-  --> $DIR/well-known-values.rs:7:7
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:26:5
    |
-LL | #[cfg(target_os = "linuz")]
-   |       ^^^^^^^^^^^^-------
-   |                   |
-   |                   help: there is a expected value with a similar name: `"linux"`
+LL |     debug_assertions = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^----------------------
+   |                     |
+   |                     help: remove the value
    |
-   = note: expected values for `target_os` are: `aix`, `android`, `cuda`, `dragonfly`, `emscripten`, `espidf`, `freebsd`, `fuchsia`, `haiku`, `hermit`, `horizon`, `hurd`, `illumos`, `ios`, `l4re`, `linux`, `macos`, `netbsd`, `none`, `nto`, `openbsd`, `psp`, `redox`, `solaris`, `solid_asp3`, `teeos`, `tvos`, `uefi`, `unknown`, `vita`, `vxworks`, `wasi`, `watchos`, `windows`, `xous`
+   = note: no expected value for `debug_assertions`
    = note: `#[warn(unexpected_cfgs)]` on by default
 
-warning: unexpected `cfg` condition value: `0`
-  --> $DIR/well-known-values.rs:14:7
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:28:5
+   |
+LL |     doc = "_UNEXPECTED_VALUE",
+   |     ^^^----------------------
+   |        |
+   |        help: remove the value
+   |
+   = note: no expected value for `doc`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:30:5
+   |
+LL |     doctest = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^----------------------
+   |            |
+   |            help: remove the value
+   |
+   = note: no expected value for `doctest`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:32:5
+   |
+LL |     miri = "_UNEXPECTED_VALUE",
+   |     ^^^^----------------------
+   |         |
+   |         help: remove the value
+   |
+   = note: no expected value for `miri`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:34:5
+   |
+LL |     overflow_checks = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^----------------------
+   |                    |
+   |                    help: remove the value
+   |
+   = note: no expected value for `overflow_checks`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:36:5
    |
-LL | #[cfg(target_has_atomic = "0")]
-   |       ^^^^^^^^^^^^^^^^^^^^---
-   |                           |
-   |                           help: there is a expected value with a similar name: `"8"`
+LL |     panic = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `panic` are: `abort`, `unwind`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:38:5
+   |
+LL |     proc_macro = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^----------------------
+   |               |
+   |               help: remove the value
+   |
+   = note: no expected value for `proc_macro`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:40:5
+   |
+LL |     relocation_model = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `relocation_model` are: `dynamic-no-pic`, `pic`, `pie`, `ropi`, `ropi-rwpi`, `rwpi`, `static`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:42:5
+   |
+LL |     sanitize = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `sanitize` are: `address`, `cfi`, `hwaddress`, `kcfi`, `kernel-address`, `leak`, `memory`, `memtag`, `safestack`, `shadow-call-stack`, `thread`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:44:5
+   |
+LL |     target_abi = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_abi` are: ``, `abi64`, `abiv2`, `abiv2hf`, `eabi`, `eabihf`, `elf`, `fortanix`, `ilp32`, `llvm`, `macabi`, `sim`, `softfloat`, `spe`, `uwp`, `vec-extabi`, `x32`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:46:5
+   |
+LL |     target_arch = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_arch` are: `aarch64`, `arm`, `avr`, `bpf`, `csky`, `hexagon`, `loongarch64`, `m68k`, `mips`, `mips32r6`, `mips64`, `mips64r6`, `msp430`, `nvptx64`, `powerpc`, `powerpc64`, `riscv32`, `riscv64`, `s390x`, `sparc`, `sparc64`, `wasm32`, `wasm64`, `x86`, `x86_64`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:48:5
+   |
+LL |     target_endian = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_endian` are: `big`, `little`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:50:5
+   |
+LL |     target_env = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_env` are: ``, `eabihf`, `gnu`, `gnueabihf`, `msvc`, `musl`, `newlib`, `nto70`, `nto71`, `ohos`, `psx`, `relibc`, `sgx`, `uclibc`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:52:5
+   |
+LL |     target_family = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_family` are: `unix`, `wasm`, `windows`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:55:5
+   |
+LL |     target_has_atomic = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = note: expected values for `target_has_atomic` are: (none), `128`, `16`, `32`, `64`, `8`, `ptr`
 
-warning: unexpected `cfg` condition value: `aa`
-  --> $DIR/well-known-values.rs:21:7
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:57:5
    |
-LL | #[cfg(unix = "aa")]
-   |       ^^^^-------
-   |           |
-   |           help: remove the value
+LL |     target_has_atomic_equal_alignment = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_has_atomic_equal_alignment` are: (none), `128`, `16`, `32`, `64`, `8`, `ptr`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:59:5
+   |
+LL |     target_has_atomic_load_store = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_has_atomic_load_store` are: (none), `128`, `16`, `32`, `64`, `8`, `ptr`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:61:5
+   |
+LL |     target_os = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_os` are: `aix`, `android`, `cuda`, `dragonfly`, `emscripten`, `espidf`, `freebsd`, `fuchsia`, `haiku`, `hermit`, `horizon`, `hurd`, `illumos`, `ios`, `l4re`, `linux`, `macos`, `netbsd`, `none`, `nto`, `openbsd`, `psp`, `redox`, `solaris`, `solid_asp3`, `teeos`, `tvos`, `uefi`, `unknown`, `vita`, `vxworks`, `wasi`, `watchos`, `windows`, `xous`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:63:5
+   |
+LL |     target_pointer_width = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_pointer_width` are: `16`, `32`, `64`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:65:5
+   |
+LL |     target_thread_local = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^----------------------
+   |                        |
+   |                        help: remove the value
+   |
+   = note: no expected value for `target_thread_local`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:67:5
+   |
+LL |     target_vendor = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: expected values for `target_vendor` are: `apple`, `espressif`, `fortanix`, `ibm`, `kmc`, `nintendo`, `nvidia`, `pc`, `sony`, `sun`, `unikraft`, `unknown`, `uwp`, `win7`, `wrs`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:69:5
+   |
+LL |     test = "_UNEXPECTED_VALUE",
+   |     ^^^^----------------------
+   |         |
+   |         help: remove the value
+   |
+   = note: no expected value for `test`
+
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:71:5
+   |
+LL |     unix = "_UNEXPECTED_VALUE",
+   |     ^^^^----------------------
+   |         |
+   |         help: remove the value
    |
    = note: no expected value for `unix`
 
-warning: unexpected `cfg` condition value: `miri`
-  --> $DIR/well-known-values.rs:28:7
+warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE`
+  --> $DIR/well-known-values.rs:73:5
    |
-LL | #[cfg(miri = "miri")]
-   |       ^^^^---------
-   |           |
-   |           help: remove the value
+LL |     windows = "_UNEXPECTED_VALUE",
+   |     ^^^^^^^----------------------
+   |            |
+   |            help: remove the value
    |
-   = note: no expected value for `miri`
+   = note: no expected value for `windows`
 
-warning: unexpected `cfg` condition value: `linux`
-  --> $DIR/well-known-values.rs:35:7
+warning: unexpected `cfg` condition value: `linuz`
+  --> $DIR/well-known-values.rs:79:7
    |
-LL | #[cfg(doc = "linux")]
-   |       ^^^----------
-   |          |
-   |          help: remove the value
+LL | #[cfg(target_os = "linuz")] // testing that we suggest `linux`
+   |       ^^^^^^^^^^^^-------
+   |                   |
+   |                   help: there is a expected value with a similar name: `"linux"`
    |
-   = note: no expected value for `doc`
+   = note: expected values for `target_os` are: `aix`, `android`, `cuda`, `dragonfly`, `emscripten`, `espidf`, `freebsd`, `fuchsia`, `haiku`, `hermit`, `horizon`, `hurd`, `illumos`, `ios`, `l4re`, `linux`, `macos`, `netbsd`, `none`, `nto`, `openbsd`, `psp`, `redox`, `solaris`, `solid_asp3`, `teeos`, `tvos`, `uefi`, `unknown`, `vita`, `vxworks`, `wasi`, `watchos`, `windows`, `xous`
 
-warning: 5 warnings emitted
+warning: 25 warnings emitted
 
-- 
cgit 1.4.1-3-g733a5


From 4c1671674e97fa8a60f3999a3770f0c32499c584 Mon Sep 17 00:00:00 2001
From: Urgau <urgau@numericable.fr>
Date: Fri, 8 Dec 2023 00:42:57 +0100
Subject: Avoid target_os and target_arch in some check-cfg tests

as they unnecessarily clutter the diagnostic output and make the
experience of adding a new target to the compiler more painful than
it should be.

target_os and target_arch are still being tested in the
well-known-values.rs test, but in one place.
---
 tests/ui/check-cfg/compact-values.rs         |  2 +-
 tests/ui/check-cfg/compact-values.stderr     |  6 +++---
 tests/ui/check-cfg/values-target-json.rs     |  4 ----
 tests/ui/check-cfg/values-target-json.stderr | 13 -------------
 4 files changed, 4 insertions(+), 21 deletions(-)
 delete mode 100644 tests/ui/check-cfg/values-target-json.stderr

diff --git a/tests/ui/check-cfg/compact-values.rs b/tests/ui/check-cfg/compact-values.rs
index 13c072fe920..80cf75d2770 100644
--- a/tests/ui/check-cfg/compact-values.rs
+++ b/tests/ui/check-cfg/compact-values.rs
@@ -8,7 +8,7 @@
 #[cfg(target(os = "linux", arch = "arm"))]
 pub fn expected() {}
 
-#[cfg(target(os = "linux", arch = "X"))]
+#[cfg(target(os = "linux", pointer_width = "X"))]
 //~^ WARNING unexpected `cfg` condition value
 pub fn unexpected() {}
 
diff --git a/tests/ui/check-cfg/compact-values.stderr b/tests/ui/check-cfg/compact-values.stderr
index bb2f4915b5e..819b789c3e5 100644
--- a/tests/ui/check-cfg/compact-values.stderr
+++ b/tests/ui/check-cfg/compact-values.stderr
@@ -1,10 +1,10 @@
 warning: unexpected `cfg` condition value: `X`
   --> $DIR/compact-values.rs:11:28
    |
-LL | #[cfg(target(os = "linux", arch = "X"))]
-   |                            ^^^^^^^^^^
+LL | #[cfg(target(os = "linux", pointer_width = "X"))]
+   |                            ^^^^^^^^^^^^^^^^^^^
    |
-   = note: expected values for `target_arch` are: `aarch64`, `arm`, `avr`, `bpf`, `csky`, `hexagon`, `loongarch64`, `m68k`, `mips`, `mips32r6`, `mips64`, `mips64r6`, `msp430`, `nvptx64`, `powerpc`, `powerpc64`, `riscv32`, `riscv64`, `s390x`, `sparc`, `sparc64`, `wasm32`, `wasm64`, `x86`, `x86_64`
+   = note: expected values for `target_pointer_width` are: `16`, `32`, `64`
    = note: `#[warn(unexpected_cfgs)]` on by default
 
 warning: 1 warning emitted
diff --git a/tests/ui/check-cfg/values-target-json.rs b/tests/ui/check-cfg/values-target-json.rs
index e4c1b54cccc..47ac79e0dbf 100644
--- a/tests/ui/check-cfg/values-target-json.rs
+++ b/tests/ui/check-cfg/values-target-json.rs
@@ -10,10 +10,6 @@
 #[lang = "sized"]
 trait Sized {}
 
-#[cfg(target_os = "linuz")]
-//~^ WARNING unexpected `cfg` condition value
-fn target_os_linux_misspell() {}
-
 #[cfg(target_os = "linux")]
 fn target_os_linux() {}
 
diff --git a/tests/ui/check-cfg/values-target-json.stderr b/tests/ui/check-cfg/values-target-json.stderr
deleted file mode 100644
index e71149f337f..00000000000
--- a/tests/ui/check-cfg/values-target-json.stderr
+++ /dev/null
@@ -1,13 +0,0 @@
-warning: unexpected `cfg` condition value: `linuz`
-  --> $DIR/values-target-json.rs:13:7
-   |
-LL | #[cfg(target_os = "linuz")]
-   |       ^^^^^^^^^^^^-------
-   |                   |
-   |                   help: there is a expected value with a similar name: `"linux"`
-   |
-   = note: expected values for `target_os` are: `aix`, `android`, `cuda`, `dragonfly`, `emscripten`, `ericos`, `espidf`, `freebsd`, `fuchsia`, `haiku`, `hermit`, `horizon`, `hurd`, `illumos`, `ios`, `l4re`, `linux`, `macos`, `netbsd`, `none`, `nto`, `openbsd`, `psp`, `redox`, `solaris`, `solid_asp3`, `teeos`, `tvos`, `uefi`, `unknown`, `vita`, `vxworks`, `wasi`, `watchos`, `windows`, `xous`
-   = note: `#[warn(unexpected_cfgs)]` on by default
-
-warning: 1 warning emitted
-
-- 
cgit 1.4.1-3-g733a5


From 97ae5095f52100b98170ab476e516d2be5b2c297 Mon Sep 17 00:00:00 2001
From: Jakub Okoński <jakub@okonski.org>
Date: Wed, 15 Nov 2023 23:08:02 +0100
Subject: Add simd_masked_{load,store} platform-intrinsics

This maps to the LLVM intrinsics: llvm.masked.load and llvm.masked.store
---
 .../rustc_codegen_cranelift/src/intrinsics/simd.rs |  52 +++++-
 compiler/rustc_codegen_llvm/src/intrinsic.rs       | 192 +++++++++++++++++++++
 compiler/rustc_hir_analysis/src/check/intrinsic.rs |   2 +
 compiler/rustc_span/src/symbol.rs                  |   2 +
 .../simd-intrinsic-generic-masked-load.rs          |  34 ++++
 .../simd-intrinsic-generic-masked-store.rs         |  32 ++++
 tests/ui/simd/masked-load-store-build-fail.rs      |  74 ++++++++
 tests/ui/simd/masked-load-store-build-fail.stderr  |  83 +++++++++
 tests/ui/simd/masked-load-store-check-fail.rs      |  32 ++++
 tests/ui/simd/masked-load-store-check-fail.stderr  |  59 +++++++
 tests/ui/simd/masked-load-store.rs                 |  33 ++++
 11 files changed, 594 insertions(+), 1 deletion(-)
 create mode 100644 tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs
 create mode 100644 tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs
 create mode 100644 tests/ui/simd/masked-load-store-build-fail.rs
 create mode 100644 tests/ui/simd/masked-load-store-build-fail.stderr
 create mode 100644 tests/ui/simd/masked-load-store-check-fail.rs
 create mode 100644 tests/ui/simd/masked-load-store-check-fail.stderr
 create mode 100644 tests/ui/simd/masked-load-store.rs

diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 0bd211fd614..5997e6026b4 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -1,5 +1,6 @@
 //! Codegen `extern "platform-intrinsic"` intrinsics.
 
+use cranelift_codegen::ir::immediates::Offset32;
 use rustc_middle::ty::GenericArgsRef;
 use rustc_span::Symbol;
 use rustc_target::abi::Endian;
@@ -1008,8 +1009,57 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
         }
 
+        sym::simd_masked_load => {
+            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
+
+            let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+            let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(val_lane_count, mask_lane_count);
+            assert_eq!(val_lane_count, ret_lane_count);
+
+            let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+            let ptr_val = ptr.load_scalar(fx);
+
+            for lane_idx in 0..ret_lane_count {
+                let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
+
+                let if_enabled = fx.bcx.create_block();
+                let if_disabled = fx.bcx.create_block();
+                let next = fx.bcx.create_block();
+                let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
+
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]);
+                fx.bcx.seal_block(if_enabled);
+                fx.bcx.seal_block(if_disabled);
+
+                fx.bcx.switch_to_block(if_enabled);
+                let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32;
+                let res = fx.bcx.ins().load(
+                    lane_clif_ty,
+                    MemFlags::trusted(),
+                    ptr_val,
+                    Offset32::new(offset),
+                );
+                fx.bcx.ins().jump(next, &[res]);
+
+                fx.bcx.switch_to_block(if_disabled);
+                fx.bcx.ins().jump(next, &[val_lane]);
+
+                fx.bcx.seal_block(next);
+                fx.bcx.switch_to_block(next);
+
+                fx.bcx.ins().nop();
+
+                ret.place_lane(fx, lane_idx)
+                    .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout));
+            }
+        }
+
         sym::simd_scatter => {
-            intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
+            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
 
             let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
             let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index cc7e78b9c62..f16014e1361 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -1492,6 +1492,198 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         return Ok(v);
     }
 
+    if name == sym::simd_masked_load {
+        // simd_masked_load(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T>
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+        // Loads contiguous elements from memory behind `pointer`, but only for
+        // those lanes whose `mask` bit is enabled.
+        // The memory addresses corresponding to the “off” lanes are not accessed.
+
+        // The element type of the "mask" argument must be a signed integer type of any width
+        let mask_ty = in_ty;
+        let (mask_len, mask_elem) = (in_len, in_elem);
+
+        // The second argument must be a pointer matching the element type
+        let pointer_ty = arg_tys[1];
+
+        // The last argument is a passthrough vector providing values for disabled lanes
+        let values_ty = arg_tys[2];
+        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
+
+        require_simd!(ret_ty, SimdReturn);
+
+        // Of the same length:
+        require!(
+            values_len == mask_len,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len: mask_len,
+                in_ty: mask_ty,
+                arg_ty: values_ty,
+                out_len: values_len
+            }
+        );
+
+        // The return type must match the last argument type
+        require!(
+            ret_ty == values_ty,
+            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
+        );
+
+        require!(
+            matches!(
+                pointer_ty.kind(),
+                ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind()
+            ),
+            InvalidMonomorphization::ExpectedElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                second_arg: pointer_ty,
+                in_elem: values_elem,
+                in_ty: values_ty,
+                mutability: ExpectedPointerMutability::Not,
+            }
+        );
+
+        require!(
+            matches!(mask_elem.kind(), ty::Int(_)),
+            InvalidMonomorphization::ThirdArgElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                third_arg: mask_ty,
+            }
+        );
+
+        // Alignment of T, must be a constant integer value:
+        let alignment_ty = bx.type_i32();
+        let alignment = bx.const_i32(bx.align_of(values_ty).bytes() as i32);
+
+        // Truncate the mask vector to a vector of i1s:
+        let (mask, mask_ty) = {
+            let i1 = bx.type_i1();
+            let i1xn = bx.type_vector(i1, mask_len);
+            (bx.trunc(args[0].immediate(), i1xn), i1xn)
+        };
+
+        let llvm_pointer = bx.type_ptr();
+
+        // Type of the vector of elements:
+        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
+        let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
+
+        let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0");
+        let fn_ty = bx
+            .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty);
+        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+        let v = bx.call(
+            fn_ty,
+            None,
+            None,
+            f,
+            &[args[1].immediate(), alignment, mask, args[2].immediate()],
+            None,
+        );
+        return Ok(v);
+    }
+
+    if name == sym::simd_masked_store {
+        // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+        // Stores contiguous elements to memory behind `pointer`, but only for
+        // those lanes whose `mask` bit is enabled.
+        // The memory addresses corresponding to the “off” lanes are not accessed.
+
+        // The element type of the "mask" argument must be a signed integer type of any width
+        let mask_ty = in_ty;
+        let (mask_len, mask_elem) = (in_len, in_elem);
+
+        // The second argument must be a pointer matching the element type
+        let pointer_ty = arg_tys[1];
+
+        // The last argument specifies the values to store to memory
+        let values_ty = arg_tys[2];
+        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
+
+        // Of the same length:
+        require!(
+            values_len == mask_len,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len: mask_len,
+                in_ty: mask_ty,
+                arg_ty: values_ty,
+                out_len: values_len
+            }
+        );
+
+        // The second argument must be a mutable pointer type matching the element type
+        require!(
+            matches!(
+                pointer_ty.kind(),
+                ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() && p.mutbl.is_mut()
+            ),
+            InvalidMonomorphization::ExpectedElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                second_arg: pointer_ty,
+                in_elem: values_elem,
+                in_ty: values_ty,
+                mutability: ExpectedPointerMutability::Mut,
+            }
+        );
+
+        require!(
+            matches!(mask_elem.kind(), ty::Int(_)),
+            InvalidMonomorphization::ThirdArgElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                third_arg: mask_ty,
+            }
+        );
+
+        // Alignment of T, must be a constant integer value:
+        let alignment_ty = bx.type_i32();
+        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
+
+        // Truncate the mask vector to a vector of i1s:
+        let (mask, mask_ty) = {
+            let i1 = bx.type_i1();
+            let i1xn = bx.type_vector(i1, in_len);
+            (bx.trunc(args[0].immediate(), i1xn), i1xn)
+        };
+
+        let ret_t = bx.type_void();
+
+        let llvm_pointer = bx.type_ptr();
+
+        // Type of the vector of elements:
+        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
+        let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
+
+        let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0");
+        let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t);
+        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+        let v = bx.call(
+            fn_ty,
+            None,
+            None,
+            f,
+            &[args[2].immediate(), args[1].immediate(), alignment, mask],
+            None,
+        );
+        return Ok(v);
+    }
+
     if name == sym::simd_scatter {
         // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
         //             mask: <N x i{M}>) -> ()
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index 7ea21b24fc8..33337190562 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -521,6 +521,8 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         sym::simd_fpowi => (1, 0, vec![param(0), tcx.types.i32], param(0)),
         sym::simd_fma => (1, 0, vec![param(0), param(0), param(0)], param(0)),
         sym::simd_gather => (3, 0, vec![param(0), param(1), param(2)], param(0)),
+        sym::simd_masked_load => (3, 0, vec![param(0), param(1), param(2)], param(2)),
+        sym::simd_masked_store => (3, 0, vec![param(0), param(1), param(2)], Ty::new_unit(tcx)),
         sym::simd_scatter => (3, 0, vec![param(0), param(1), param(2)], Ty::new_unit(tcx)),
         sym::simd_insert => (2, 0, vec![param(0), tcx.types.u32, param(1)], param(0)),
         sym::simd_extract => (2, 0, vec![param(0), tcx.types.u32], param(1)),
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 7b9b7b85293..485265e6889 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1516,6 +1516,8 @@ symbols! {
         simd_insert,
         simd_le,
         simd_lt,
+        simd_masked_load,
+        simd_masked_store,
         simd_mul,
         simd_ne,
         simd_neg,
diff --git a/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs
new file mode 100644
index 00000000000..7b1fb320894
--- /dev/null
+++ b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs
@@ -0,0 +1,34 @@
+// compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+
+#![feature(repr_simd, platform_intrinsics)]
+#![allow(non_camel_case_types)]
+
+#[repr(simd)]
+#[derive(Copy, Clone, PartialEq, Debug)]
+pub struct Vec2<T>(pub T, pub T);
+
+#[repr(simd)]
+#[derive(Copy, Clone, PartialEq, Debug)]
+pub struct Vec4<T>(pub T, pub T, pub T, pub T);
+
+extern "platform-intrinsic" {
+    fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
+}
+
+// CHECK-LABEL: @load_f32x2
+#[no_mangle]
+pub unsafe fn load_f32x2(mask: Vec2<i32>, pointer: *const f32,
+                         values: Vec2<f32>) -> Vec2<f32> {
+    // CHECK: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 {{.*}}, <2 x i1> {{.*}}, <2 x float> {{.*}})
+    simd_masked_load(mask, pointer, values)
+}
+
+// CHECK-LABEL: @load_pf32x4
+#[no_mangle]
+pub unsafe fn load_pf32x4(mask: Vec4<i32>, pointer: *const *const f32,
+                          values: Vec4<*const f32>) -> Vec4<*const f32> {
+    // CHECK: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr {{.*}}, i32 {{.*}}, <4 x i1> {{.*}}, <4 x ptr> {{.*}})
+    simd_masked_load(mask, pointer, values)
+}
diff --git a/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs
new file mode 100644
index 00000000000..d8a37020f23
--- /dev/null
+++ b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs
@@ -0,0 +1,32 @@
+// compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+
+#![feature(repr_simd, platform_intrinsics)]
+#![allow(non_camel_case_types)]
+
+#[repr(simd)]
+#[derive(Copy, Clone, PartialEq, Debug)]
+pub struct Vec2<T>(pub T, pub T);
+
+#[repr(simd)]
+#[derive(Copy, Clone, PartialEq, Debug)]
+pub struct Vec4<T>(pub T, pub T, pub T, pub T);
+
+extern "platform-intrinsic" {
+    fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T) -> ();
+}
+
+// CHECK-LABEL: @store_f32x2
+#[no_mangle]
+pub unsafe fn store_f32x2(mask: Vec2<i32>, pointer: *mut f32, values: Vec2<f32>) {
+    // CHECK: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 {{.*}}, <2 x i1> {{.*}})
+    simd_masked_store(mask, pointer, values)
+}
+
+// CHECK-LABEL: @store_pf32x4
+#[no_mangle]
+pub unsafe fn store_pf32x4(mask: Vec4<i32>, pointer: *mut *const f32, values: Vec4<*const f32>) {
+    // CHECK: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr {{.*}}, i32 {{.*}}, <4 x i1> {{.*}})
+    simd_masked_store(mask, pointer, values)
+}
diff --git a/tests/ui/simd/masked-load-store-build-fail.rs b/tests/ui/simd/masked-load-store-build-fail.rs
new file mode 100644
index 00000000000..9b79b3bd6ea
--- /dev/null
+++ b/tests/ui/simd/masked-load-store-build-fail.rs
@@ -0,0 +1,74 @@
+// build-fail
+#![feature(repr_simd, platform_intrinsics)]
+
+extern "platform-intrinsic" {
+    fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
+    fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T) -> ();
+}
+
+#[derive(Copy, Clone)]
+#[repr(simd)]
+struct Simd<T, const N: usize>([T; N]);
+
+fn main() {
+    unsafe {
+        let mut arr = [4u8, 5, 6, 7];
+        let default = Simd::<u8, 4>([9; 4]);
+
+        simd_masked_load(
+            Simd::<i8, 8>([-1, 0, -1, -1, 0, 0, 0, 0]),
+            arr.as_ptr(),
+            default
+        );
+        //~^^^^^ ERROR expected third argument with length 8 (same as input type `Simd<i8, 8>`), found `Simd<u8, 4>` with length 4
+
+        simd_masked_load(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            arr.as_ptr() as *const i8,
+            default
+        );
+        //~^^^^^ ERROR expected element type `u8` of second argument `*const i8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*_ u8`
+
+        simd_masked_load(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            arr.as_ptr(),
+            Simd::<u32, 4>([9; 4])
+        );
+        //~^^^^^ ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*_ u32`
+
+        simd_masked_load(
+            Simd::<u8, 4>([1, 0, 1, 1]),
+            arr.as_ptr(),
+            default
+        );
+        //~^^^^^ ERROR expected element type `u8` of third argument `Simd<u8, 4>` to be a signed integer type
+
+        simd_masked_store(
+            Simd([-1i8; 4]),
+            arr.as_ptr(),
+            Simd([5u32; 4])
+        );
+        //~^^^^^ ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*mut u32`
+
+        simd_masked_store(
+            Simd([-1i8; 4]),
+            arr.as_ptr(),
+            Simd([5u8; 4])
+        );
+        //~^^^^^ ERROR expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
+
+        simd_masked_store(
+            Simd([-1i8; 4]),
+            arr.as_mut_ptr(),
+            Simd([5u8; 2])
+        );
+        //~^^^^^ ERROR expected third argument with length 4 (same as input type `Simd<i8, 4>`), found `Simd<u8, 2>` with length 2
+
+        simd_masked_store(
+            Simd([1u32; 4]),
+            arr.as_mut_ptr(),
+            Simd([5u8; 4])
+        );
+        //~^^^^^ ERROR expected element type `u8` of third argument `Simd<u32, 4>` to be a signed integer type
+    }
+}
diff --git a/tests/ui/simd/masked-load-store-build-fail.stderr b/tests/ui/simd/masked-load-store-build-fail.stderr
new file mode 100644
index 00000000000..59af83fe0e8
--- /dev/null
+++ b/tests/ui/simd/masked-load-store-build-fail.stderr
@@ -0,0 +1,83 @@
+error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected third argument with length 8 (same as input type `Simd<i8, 8>`), found `Simd<u8, 4>` with length 4
+  --> $DIR/masked-load-store-build-fail.rs:18:9
+   |
+LL | /         simd_masked_load(
+LL | |             Simd::<i8, 8>([-1, 0, -1, -1, 0, 0, 0, 0]),
+LL | |             arr.as_ptr(),
+LL | |             default
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u8` of second argument `*const i8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*_ u8`
+  --> $DIR/masked-load-store-build-fail.rs:25:9
+   |
+LL | /         simd_masked_load(
+LL | |             Simd::<i8, 4>([-1, 0, -1, -1]),
+LL | |             arr.as_ptr() as *const i8,
+LL | |             default
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*_ u32`
+  --> $DIR/masked-load-store-build-fail.rs:32:9
+   |
+LL | /         simd_masked_load(
+LL | |             Simd::<i8, 4>([-1, 0, -1, -1]),
+LL | |             arr.as_ptr(),
+LL | |             Simd::<u32, 4>([9; 4])
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u8` of third argument `Simd<u8, 4>` to be a signed integer type
+  --> $DIR/masked-load-store-build-fail.rs:39:9
+   |
+LL | /         simd_masked_load(
+LL | |             Simd::<u8, 4>([1, 0, 1, 1]),
+LL | |             arr.as_ptr(),
+LL | |             default
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*mut u32`
+  --> $DIR/masked-load-store-build-fail.rs:46:9
+   |
+LL | /         simd_masked_store(
+LL | |             Simd([-1i8; 4]),
+LL | |             arr.as_ptr(),
+LL | |             Simd([5u32; 4])
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
+  --> $DIR/masked-load-store-build-fail.rs:53:9
+   |
+LL | /         simd_masked_store(
+LL | |             Simd([-1i8; 4]),
+LL | |             arr.as_ptr(),
+LL | |             Simd([5u8; 4])
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected third argument with length 4 (same as input type `Simd<i8, 4>`), found `Simd<u8, 2>` with length 2
+  --> $DIR/masked-load-store-build-fail.rs:60:9
+   |
+LL | /         simd_masked_store(
+LL | |             Simd([-1i8; 4]),
+LL | |             arr.as_mut_ptr(),
+LL | |             Simd([5u8; 2])
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u8` of third argument `Simd<u32, 4>` to be a signed integer type
+  --> $DIR/masked-load-store-build-fail.rs:67:9
+   |
+LL | /         simd_masked_store(
+LL | |             Simd([1u32; 4]),
+LL | |             arr.as_mut_ptr(),
+LL | |             Simd([5u8; 4])
+LL | |         );
+   | |_________^
+
+error: aborting due to 8 previous errors
+
+For more information about this error, try `rustc --explain E0511`.
diff --git a/tests/ui/simd/masked-load-store-check-fail.rs b/tests/ui/simd/masked-load-store-check-fail.rs
new file mode 100644
index 00000000000..d4b35e211c8
--- /dev/null
+++ b/tests/ui/simd/masked-load-store-check-fail.rs
@@ -0,0 +1,32 @@
+// check-fail
+#![feature(repr_simd, platform_intrinsics)]
+
+extern "platform-intrinsic" {
+    fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
+    fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T) -> ();
+}
+
+#[derive(Copy, Clone)]
+#[repr(simd)]
+struct Simd<T, const N: usize>([T; N]);
+
+fn main() {
+    unsafe {
+        let mut arr = [4u8, 5, 6, 7];
+        let default = Simd::<u8, 4>([9; 4]);
+
+        let _x: Simd<u8, 2> = simd_masked_load(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            arr.as_ptr(),
+            Simd::<u8, 4>([9; 4])
+        );
+        //~^^ ERROR mismatched types
+
+        let _x: Simd<u32, 4> = simd_masked_load(
+            Simd::<u8, 4>([1, 0, 1, 1]),
+            arr.as_ptr(),
+            default
+        );
+        //~^^ ERROR mismatched types
+    }
+}
diff --git a/tests/ui/simd/masked-load-store-check-fail.stderr b/tests/ui/simd/masked-load-store-check-fail.stderr
new file mode 100644
index 00000000000..5d205d607c9
--- /dev/null
+++ b/tests/ui/simd/masked-load-store-check-fail.stderr
@@ -0,0 +1,59 @@
+error[E0308]: mismatched types
+  --> $DIR/masked-load-store-check-fail.rs:21:13
+   |
+LL |         let _x: Simd<u8, 2> = simd_masked_load(
+   |                               ---------------- arguments to this function are incorrect
+...
+LL |             Simd::<u8, 4>([9; 4])
+   |             ^^^^^^^^^^^^^^^^^^^^^ expected `2`, found `4`
+   |
+   = note: expected struct `Simd<_, 2>`
+              found struct `Simd<_, 4>`
+help: the return type of this call is `Simd<u8, 4>` due to the type of the argument passed
+  --> $DIR/masked-load-store-check-fail.rs:18:31
+   |
+LL |           let _x: Simd<u8, 2> = simd_masked_load(
+   |  _______________________________^
+LL | |             Simd::<i8, 4>([-1, 0, -1, -1]),
+LL | |             arr.as_ptr(),
+LL | |             Simd::<u8, 4>([9; 4])
+   | |             --------------------- this argument influences the return type of `simd_masked_load`
+LL | |         );
+   | |_________^
+note: function defined here
+  --> $DIR/masked-load-store-check-fail.rs:5:8
+   |
+LL |     fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
+   |        ^^^^^^^^^^^^^^^^
+
+error[E0308]: mismatched types
+  --> $DIR/masked-load-store-check-fail.rs:28:13
+   |
+LL |         let _x: Simd<u32, 4> = simd_masked_load(
+   |                                ---------------- arguments to this function are incorrect
+...
+LL |             default
+   |             ^^^^^^^ expected `Simd<u32, 4>`, found `Simd<u8, 4>`
+   |
+   = note: expected struct `Simd<u32, _>`
+              found struct `Simd<u8, _>`
+help: the return type of this call is `Simd<u8, 4>` due to the type of the argument passed
+  --> $DIR/masked-load-store-check-fail.rs:25:32
+   |
+LL |           let _x: Simd<u32, 4> = simd_masked_load(
+   |  ________________________________^
+LL | |             Simd::<u8, 4>([1, 0, 1, 1]),
+LL | |             arr.as_ptr(),
+LL | |             default
+   | |             ------- this argument influences the return type of `simd_masked_load`
+LL | |         );
+   | |_________^
+note: function defined here
+  --> $DIR/masked-load-store-check-fail.rs:5:8
+   |
+LL |     fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
+   |        ^^^^^^^^^^^^^^^^
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0308`.
diff --git a/tests/ui/simd/masked-load-store.rs b/tests/ui/simd/masked-load-store.rs
new file mode 100644
index 00000000000..74ee652ec6e
--- /dev/null
+++ b/tests/ui/simd/masked-load-store.rs
@@ -0,0 +1,33 @@
+// run-pass
+#![feature(repr_simd, platform_intrinsics)]
+
+extern "platform-intrinsic" {
+    fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
+    fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T) -> ();
+}
+
+#[derive(Copy, Clone)]
+#[repr(simd)]
+struct Simd<T, const N: usize>([T; N]);
+
+fn main() {
+    unsafe {
+        let a = Simd::<u8, 4>([0, 1, 2, 3]);
+        let b_src = [4u8, 5, 6, 7];
+        let b_default = Simd::<u8, 4>([9; 4]);
+        let b: Simd::<u8, 4> = simd_masked_load(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            b_src.as_ptr(),
+            b_default
+        );
+
+        assert_eq!(&b.0, &[4, 9, 6, 7]);
+
+        let mut output = [u8::MAX; 5];
+
+        simd_masked_store(Simd::<i8, 4>([-1, -1, -1, 0]), output.as_mut_ptr(), a);
+        assert_eq!(&output, &[0, 1, 2, u8::MAX, u8::MAX]);
+        simd_masked_store(Simd::<i8, 4>([0, -1, -1, 0]), output[1..].as_mut_ptr(), b);
+        assert_eq!(&output, &[0, 1, 9, 6, u8::MAX]);
+    }
+}
-- 
cgit 1.4.1-3-g733a5


From 2cf54e9f99f83e314256808b1e0753779d49e652 Mon Sep 17 00:00:00 2001
From: jyn <github@jyn.dev>
Date: Sat, 9 Dec 2023 09:43:27 -0500
Subject: use `&` instead of start-process in x.ps1

start-process has weird parsing rules and buggy behavior. we've already had to work around them several times, and the workarounds were not complete.
i wonder who could have added it HMMMMMM
```
PS C:\Users\jyn\src\rust> git log --reverse -S Start-Process x.ps1
commit 775c3c0493e9a383a7f1c521b06d36f2e3d0d886
Author: Jynn Nelson <github@jyn.dev>
Date:   Sun Jul 31 14:02:31 2022 -0500

    Add `x.sh` and `x.ps1` shell scripts
```

the latest broken thing is trailing backslashes:
```
$ x.ps1 t .\tests\ui\error-emitter\
```
would be transformed into
```
['t', '.\\tests\\ui\\error-emitter"']
```

rather than trying to hack around that too, abandon start-process altogether and just use `&`.
---
 x.ps1 | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/x.ps1 b/x.ps1
index eae1a2cb399..afd988e67e4 100755
--- a/x.ps1
+++ b/x.ps1
@@ -8,12 +8,7 @@ $ErrorActionPreference = "Stop"
 Get-Command -syntax ${PSCommandPath} >$null
 
 $xpy = Join-Path $PSScriptRoot x.py
-# Start-Process for some reason splits arguments on spaces. (Isn't powershell supposed to be simpler than bash?)
-# Double-quote all the arguments so it doesn't do that.
-$xpy_args = @("""$xpy""")
-foreach ($arg in $args) {
-    $xpy_args += """$arg"""
-}
+$xpy_args = @($xpy) + $args
 
 function Get-Application($app) {
     $cmd = Get-Command $app -ErrorAction SilentlyContinue -CommandType Application | Select-Object -First 1
@@ -21,16 +16,8 @@ function Get-Application($app) {
 }
 
 function Invoke-Application($application, $arguments) {
-    $process = Start-Process -NoNewWindow -PassThru $application $arguments
-    # WORKAROUND: Caching the handle is necessary to make ExitCode work.
-    # See https://stackoverflow.com/a/23797762
-    $handle = $process.Handle
-    $process.WaitForExit()
-    if ($null -eq $process.ExitCode) {
-        Write-Error "Unable to read the exit code"
-        Exit 1
-    }
-    Exit $process.ExitCode
+    & $application $arguments
+    Exit $LASTEXITCODE
 }
 
 foreach ($python in "py", "python3", "python", "python2") {
-- 
cgit 1.4.1-3-g733a5


From 6860654d82963fa2f4d0f1ef6adaf7a1d82f80df Mon Sep 17 00:00:00 2001
From: onur-ozkan <work@onurozkan.dev>
Date: Sun, 19 Nov 2023 22:23:54 +0300
Subject: allow bypassing the build directory lock

As bootstrap locks its entire build directory, parallel bootstrapping
for anything becomes impossible. This change enables developers to bypass
the locking mechanism when it is unnecessary for their specific use case.

Signed-off-by: onur-ozkan <work@onurozkan.dev>
---
 src/bootstrap/src/bin/main.rs           | 61 ++++++++++++++++++---------------
 src/bootstrap/src/core/config/config.rs |  3 ++
 src/bootstrap/src/core/config/flags.rs  |  7 ++++
 src/etc/completions/x.py.fish           | 25 +++++++++++---
 src/etc/completions/x.py.ps1            | 35 +++++++++++++------
 src/etc/completions/x.py.sh             | 30 ++++++++--------
 src/etc/completions/x.py.zsh            | 35 +++++++++++++------
 7 files changed, 128 insertions(+), 68 deletions(-)

diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs
index 0a6072ae1a5..84eefc32b2f 100644
--- a/src/bootstrap/src/bin/main.rs
+++ b/src/bootstrap/src/bin/main.rs
@@ -23,35 +23,40 @@ fn main() {
     let mut build_lock;
     #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
     let _build_lock_guard;
-    #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
-    // Display PID of process holding the lock
-    // PID will be stored in a lock file
-    {
-        let path = config.out.join("lock");
-        let pid = match fs::read_to_string(&path) {
-            Ok(contents) => contents,
-            Err(_) => String::new(),
-        };
-
-        build_lock =
-            fd_lock::RwLock::new(t!(fs::OpenOptions::new().write(true).create(true).open(&path)));
-        _build_lock_guard = match build_lock.try_write() {
-            Ok(mut lock) => {
-                t!(lock.write(&process::id().to_string().as_ref()));
-                lock
-            }
-            err => {
-                drop(err);
-                println!("WARNING: build directory locked by process {pid}, waiting for lock");
-                let mut lock = t!(build_lock.write());
-                t!(lock.write(&process::id().to_string().as_ref()));
-                lock
-            }
-        };
-    }
 
-    #[cfg(any(not(any(unix, windows)), target_os = "solaris"))]
-    println!("WARNING: file locking not supported for target, not locking build directory");
+    if !config.bypass_bootstrap_lock {
+        // Display PID of process holding the lock
+        // PID will be stored in a lock file
+        #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
+        {
+            let path = config.out.join("lock");
+            let pid = match fs::read_to_string(&path) {
+                Ok(contents) => contents,
+                Err(_) => String::new(),
+            };
+
+            build_lock = fd_lock::RwLock::new(t!(fs::OpenOptions::new()
+                .write(true)
+                .create(true)
+                .open(&path)));
+            _build_lock_guard = match build_lock.try_write() {
+                Ok(mut lock) => {
+                    t!(lock.write(&process::id().to_string().as_ref()));
+                    lock
+                }
+                err => {
+                    drop(err);
+                    println!("WARNING: build directory locked by process {pid}, waiting for lock");
+                    let mut lock = t!(build_lock.write());
+                    t!(lock.write(&process::id().to_string().as_ref()));
+                    lock
+                }
+            };
+        }
+
+        #[cfg(any(not(any(unix, windows)), target_os = "solaris"))]
+        println!("WARNING: file locking not supported for target, not locking build directory");
+    }
 
     // check_version warnings are not printed during setup
     let changelog_suggestion =
diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index 0a9175aa3ea..b772a95b655 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -117,6 +117,7 @@ impl Display for DebuginfoLevel {
 pub struct Config {
     pub changelog_seen: Option<usize>, // FIXME: Deprecated field. Remove it at 2024.
     pub change_id: Option<usize>,
+    pub bypass_bootstrap_lock: bool,
     pub ccache: Option<String>,
     /// Call Build::ninja() instead of this.
     pub ninja_in_file: bool,
@@ -1057,6 +1058,7 @@ define_config! {
 impl Config {
     pub fn default_opts() -> Config {
         let mut config = Config::default();
+        config.bypass_bootstrap_lock = false;
         config.llvm_optimize = true;
         config.ninja_in_file = true;
         config.llvm_static_stdcpp = false;
@@ -1135,6 +1137,7 @@ impl Config {
         config.llvm_profile_use = flags.llvm_profile_use;
         config.llvm_profile_generate = flags.llvm_profile_generate;
         config.enable_bolt_settings = flags.enable_bolt_settings;
+        config.bypass_bootstrap_lock = flags.bypass_bootstrap_lock;
 
         // Infer the rest of the configuration.
 
diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs
index 2a301007ab4..27d0425df54 100644
--- a/src/bootstrap/src/core/config/flags.rs
+++ b/src/bootstrap/src/core/config/flags.rs
@@ -133,6 +133,13 @@ pub struct Flags {
     /// whether to use color in cargo and rustc output
     pub color: Color,
 
+    #[arg(global(true), long)]
+    /// Bootstrap uses this value to decide whether it should bypass locking the build process.
+    /// This is rarely needed (e.g., compiling the std library for different targets in parallel).
+    ///
+    /// Unless you know exactly what you are doing, you probably don't need this.
+    pub bypass_bootstrap_lock: bool,
+
     /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml
     #[arg(global(true), long, value_name = "VALUE")]
     pub llvm_skip_rebuild: Option<bool>,
diff --git a/src/etc/completions/x.py.fish b/src/etc/completions/x.py.fish
index 48796988868..99096307039 100644
--- a/src/etc/completions/x.py.fish
+++ b/src/etc/completions/x.py.fish
@@ -26,10 +26,11 @@ complete -c x.py -n "__fish_use_subcommand" -s i -l incremental -d 'use incremen
 complete -c x.py -n "__fish_use_subcommand" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_use_subcommand" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_use_subcommand" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_use_subcommand" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_use_subcommand" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_use_subcommand" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_use_subcommand" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
-complete -c x.py -n "__fish_use_subcommand" -s h -l help -d 'Print help'
+complete -c x.py -n "__fish_use_subcommand" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_use_subcommand" -f -a "build" -d 'Compile either the compiler or libraries'
 complete -c x.py -n "__fish_use_subcommand" -f -a "check" -d 'Compile either the compiler or libraries, using cargo check'
 complete -c x.py -n "__fish_use_subcommand" -f -a "clippy" -d 'Run Clippy (uses rustup/cargo-installed clippy binary)'
@@ -72,6 +73,7 @@ complete -c x.py -n "__fish_seen_subcommand_from build" -s i -l incremental -d '
 complete -c x.py -n "__fish_seen_subcommand_from build" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from build" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -105,6 +107,7 @@ complete -c x.py -n "__fish_seen_subcommand_from check" -s i -l incremental -d '
 complete -c x.py -n "__fish_seen_subcommand_from check" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from check" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -142,6 +145,7 @@ complete -c x.py -n "__fish_seen_subcommand_from clippy" -s i -l incremental -d
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from clippy" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -174,6 +178,7 @@ complete -c x.py -n "__fish_seen_subcommand_from fix" -s i -l incremental -d 'us
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from fix" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -207,6 +212,7 @@ complete -c x.py -n "__fish_seen_subcommand_from fmt" -s i -l incremental -d 'us
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from fmt" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -241,6 +247,7 @@ complete -c x.py -n "__fish_seen_subcommand_from doc" -s i -l incremental -d 'us
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from doc" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -286,6 +293,7 @@ complete -c x.py -n "__fish_seen_subcommand_from test" -s i -l incremental -d 'u
 complete -c x.py -n "__fish_seen_subcommand_from test" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from test" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -319,10 +327,11 @@ complete -c x.py -n "__fish_seen_subcommand_from bench" -s i -l incremental -d '
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from bench" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
-complete -c x.py -n "__fish_seen_subcommand_from bench" -s h -l help -d 'Print help'
+complete -c x.py -n "__fish_seen_subcommand_from bench" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l stage -d 'Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used' -r
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -352,10 +361,11 @@ complete -c x.py -n "__fish_seen_subcommand_from clean" -s i -l incremental -d '
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from clean" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
-complete -c x.py -n "__fish_seen_subcommand_from clean" -s h -l help -d 'Print help'
+complete -c x.py -n "__fish_seen_subcommand_from clean" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l build -d 'build target of the stage0 compiler' -r -f
@@ -384,10 +394,11 @@ complete -c x.py -n "__fish_seen_subcommand_from dist" -s i -l incremental -d 'u
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from dist" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
-complete -c x.py -n "__fish_seen_subcommand_from dist" -s h -l help -d 'Print help'
+complete -c x.py -n "__fish_seen_subcommand_from dist" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from install" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
 complete -c x.py -n "__fish_seen_subcommand_from install" -l build -d 'build target of the stage0 compiler' -r -f
@@ -416,10 +427,11 @@ complete -c x.py -n "__fish_seen_subcommand_from install" -s i -l incremental -d
 complete -c x.py -n "__fish_seen_subcommand_from install" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from install" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
-complete -c x.py -n "__fish_seen_subcommand_from install" -s h -l help -d 'Print help'
+complete -c x.py -n "__fish_seen_subcommand_from install" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l args -d 'arguments for the tool' -r
 complete -c x.py -n "__fish_seen_subcommand_from run" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from run" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -449,6 +461,7 @@ complete -c x.py -n "__fish_seen_subcommand_from run" -s i -l incremental -d 'us
 complete -c x.py -n "__fish_seen_subcommand_from run" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from run" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -481,6 +494,7 @@ complete -c x.py -n "__fish_seen_subcommand_from setup" -s i -l incremental -d '
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from setup" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
@@ -514,6 +528,7 @@ complete -c x.py -n "__fish_seen_subcommand_from suggest" -s i -l incremental -d
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l include-default-paths -d 'include default paths in addition to the provided ones'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l dry-run -d 'dry run; don\'t build anything'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from suggest" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
diff --git a/src/etc/completions/x.py.ps1 b/src/etc/completions/x.py.ps1
index 2fed1be7269..7e926d10b3f 100644
--- a/src/etc/completions/x.py.ps1
+++ b/src/etc/completions/x.py.ps1
@@ -52,11 +52,12 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
-            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
-            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('build', 'build', [CompletionResultType]::ParameterValue, 'Compile either the compiler or libraries')
             [CompletionResult]::new('check', 'check', [CompletionResultType]::ParameterValue, 'Compile either the compiler or libraries, using cargo check')
             [CompletionResult]::new('clippy', 'clippy', [CompletionResultType]::ParameterValue, 'Run Clippy (uses rustup/cargo-installed clippy binary)')
@@ -105,6 +106,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -145,6 +147,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -189,6 +192,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -228,6 +232,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -268,6 +273,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -309,6 +315,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -361,6 +368,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -401,11 +409,12 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
-            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
-            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
         }
         'x.py;clean' {
@@ -441,11 +450,12 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
-            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
-            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
         }
         'x.py;dist' {
@@ -480,11 +490,12 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
-            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
-            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
         }
         'x.py;install' {
@@ -519,11 +530,12 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
-            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
-            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
         }
         'x.py;run' {
@@ -559,6 +571,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -598,6 +611,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
@@ -638,6 +652,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
             [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
             [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
diff --git a/src/etc/completions/x.py.sh b/src/etc/completions/x.py.sh
index f22d7e3e131..78754f6e7b2 100644
--- a/src/etc/completions/x.py.sh
+++ b/src/etc/completions/x.py.sh
@@ -61,7 +61,7 @@ _x.py() {
 
     case "${cmd}" in
         x.py)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test bench clean dist install run setup suggest"
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test bench clean dist install run setup suggest"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -171,7 +171,7 @@ _x.py() {
             return 0
             ;;
         x.py__bench)
-            opts="-v -i -j -h --test-args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --test-args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -285,7 +285,7 @@ _x.py() {
             return 0
             ;;
         x.py__build)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -395,7 +395,7 @@ _x.py() {
             return 0
             ;;
         x.py__check)
-            opts="-v -i -j -h --all-targets --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --all-targets --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -505,7 +505,7 @@ _x.py() {
             return 0
             ;;
         x.py__clean)
-            opts="-v -i -j -h --all --stage --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --all --stage --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -615,7 +615,7 @@ _x.py() {
             return 0
             ;;
         x.py__clippy)
-            opts="-A -D -W -F -v -i -j -h --fix --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-A -D -W -F -v -i -j -h --fix --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -741,7 +741,7 @@ _x.py() {
             return 0
             ;;
         x.py__dist)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -851,7 +851,7 @@ _x.py() {
             return 0
             ;;
         x.py__doc)
-            opts="-v -i -j -h --open --json --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --open --json --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -961,7 +961,7 @@ _x.py() {
             return 0
             ;;
         x.py__fix)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1071,7 +1071,7 @@ _x.py() {
             return 0
             ;;
         x.py__fmt)
-            opts="-v -i -j -h --check --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --check --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1181,7 +1181,7 @@ _x.py() {
             return 0
             ;;
         x.py__install)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1291,7 +1291,7 @@ _x.py() {
             return 0
             ;;
         x.py__run)
-            opts="-v -i -j -h --args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1405,7 +1405,7 @@ _x.py() {
             return 0
             ;;
         x.py__setup)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [<PROFILE>|hook|vscode|link] [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [<PROFILE>|hook|vscode|link] [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1515,7 +1515,7 @@ _x.py() {
             return 0
             ;;
         x.py__suggest)
-            opts="-v -i -j -h --run --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --run --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1625,7 +1625,7 @@ _x.py() {
             return 0
             ;;
         x.py__test)
-            opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
diff --git a/src/etc/completions/x.py.zsh b/src/etc/completions/x.py.zsh
index 1e5a7b5aa89..29dfc292615 100644
--- a/src/etc/completions/x.py.zsh
+++ b/src/etc/completions/x.py.zsh
@@ -46,11 +46,12 @@ _x.py() {
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '::paths -- paths for the subcommand:_files' \
 '::free_args -- arguments passed to subcommands:' \
 ":: :_x.py_commands" \
@@ -95,6 +96,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -137,6 +139,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -183,6 +186,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -224,6 +228,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -266,6 +271,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -309,6 +315,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -363,6 +370,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -405,11 +413,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -447,11 +456,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -488,11 +498,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -529,11 +540,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -571,6 +583,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -612,6 +625,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -655,6 +669,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-- 
cgit 1.4.1-3-g733a5


From e8a25b0723b2d4e693881ecd4c1d4c7f8ba8eb51 Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Sun, 10 Dec 2023 09:03:44 +0100
Subject: Preparing for merge from rustc

---
 src/tools/miri/rust-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/miri/rust-version b/src/tools/miri/rust-version
index 8841b0c95cb..73e201aa309 100644
--- a/src/tools/miri/rust-version
+++ b/src/tools/miri/rust-version
@@ -1 +1 @@
-d6fa38a9b2426487e010a6c16862132f89755e41
+61afc9c92896a43fce92bd5e3bba6274c5e3e960
-- 
cgit 1.4.1-3-g733a5


From 281002d42c3465319181a0af50d7068935477be2 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sun, 10 Dec 2023 20:42:30 +0100
Subject: Extract exhaustiveness into its own crate

---
 Cargo.lock                                         |   22 +
 compiler/rustc_driver_impl/Cargo.toml              |    1 +
 compiler/rustc_driver_impl/src/lib.rs              |    1 +
 compiler/rustc_mir_build/Cargo.toml                |    1 +
 compiler/rustc_mir_build/messages.ftl              |   20 -
 compiler/rustc_mir_build/src/errors.rs             |   95 +-
 .../src/thir/pattern/check_match.rs                |    6 +-
 .../src/thir/pattern/deconstruct_pat.rs            | 1964 --------------------
 compiler/rustc_mir_build/src/thir/pattern/mod.rs   |    3 -
 .../rustc_mir_build/src/thir/pattern/usefulness.rs | 1648 ----------------
 compiler/rustc_pattern_analysis/Cargo.toml         |   22 +
 compiler/rustc_pattern_analysis/messages.ftl       |   19 +
 compiler/rustc_pattern_analysis/src/constructor.rs | 1241 +++++++++++++
 compiler/rustc_pattern_analysis/src/errors.rs      |   95 +
 compiler/rustc_pattern_analysis/src/lib.rs         |   13 +
 compiler/rustc_pattern_analysis/src/pat.rs         |  744 ++++++++
 compiler/rustc_pattern_analysis/src/usefulness.rs  | 1646 ++++++++++++++++
 17 files changed, 3811 insertions(+), 3730 deletions(-)
 delete mode 100644 compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
 delete mode 100644 compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
 create mode 100644 compiler/rustc_pattern_analysis/Cargo.toml
 create mode 100644 compiler/rustc_pattern_analysis/messages.ftl
 create mode 100644 compiler/rustc_pattern_analysis/src/constructor.rs
 create mode 100644 compiler/rustc_pattern_analysis/src/errors.rs
 create mode 100644 compiler/rustc_pattern_analysis/src/lib.rs
 create mode 100644 compiler/rustc_pattern_analysis/src/pat.rs
 create mode 100644 compiler/rustc_pattern_analysis/src/usefulness.rs

diff --git a/Cargo.lock b/Cargo.lock
index 6b9bb721e01..9f82b2e1640 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3756,6 +3756,7 @@ dependencies = [
  "rustc_monomorphize",
  "rustc_parse",
  "rustc_passes",
+ "rustc_pattern_analysis",
  "rustc_privacy",
  "rustc_query_system",
  "rustc_resolve",
@@ -4229,6 +4230,7 @@ dependencies = [
  "rustc_infer",
  "rustc_macros",
  "rustc_middle",
+ "rustc_pattern_analysis",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -4364,6 +4366,26 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "rustc_pattern_analysis"
+version = "0.0.0"
+dependencies = [
+ "rustc_apfloat",
+ "rustc_arena",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_fluent_macro",
+ "rustc_hir",
+ "rustc_index",
+ "rustc_macros",
+ "rustc_middle",
+ "rustc_session",
+ "rustc_span",
+ "rustc_target",
+ "smallvec",
+ "tracing",
+]
+
 [[package]]
 name = "rustc_privacy"
 version = "0.0.0"
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
index f2a8c54b6d5..49042984553 100644
--- a/compiler/rustc_driver_impl/Cargo.toml
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -38,6 +38,7 @@ rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_monomorphize = { path = "../rustc_monomorphize" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_passes = { path = "../rustc_passes" }
+rustc_pattern_analysis = { path = "../rustc_pattern_analysis" }
 rustc_privacy = { path = "../rustc_privacy" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_resolve = { path = "../rustc_resolve" }
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 1f60400b513..8b7a4dbff9d 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -128,6 +128,7 @@ pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
     rustc_monomorphize::DEFAULT_LOCALE_RESOURCE,
     rustc_parse::DEFAULT_LOCALE_RESOURCE,
     rustc_passes::DEFAULT_LOCALE_RESOURCE,
+    rustc_pattern_analysis::DEFAULT_LOCALE_RESOURCE,
     rustc_privacy::DEFAULT_LOCALE_RESOURCE,
     rustc_query_system::DEFAULT_LOCALE_RESOURCE,
     rustc_resolve::DEFAULT_LOCALE_RESOURCE,
diff --git a/compiler/rustc_mir_build/Cargo.toml b/compiler/rustc_mir_build/Cargo.toml
index db542234052..6d681dc295e 100644
--- a/compiler/rustc_mir_build/Cargo.toml
+++ b/compiler/rustc_mir_build/Cargo.toml
@@ -17,6 +17,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_pattern_analysis = { path = "../rustc_pattern_analysis" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_mir_build/messages.ftl b/compiler/rustc_mir_build/messages.ftl
index c8d6c2114e9..615b553434f 100644
--- a/compiler/rustc_mir_build/messages.ftl
+++ b/compiler/rustc_mir_build/messages.ftl
@@ -237,15 +237,6 @@ mir_build_non_const_path = runtime values cannot be referenced in patterns
 mir_build_non_exhaustive_match_all_arms_guarded =
     match arms with guards don't count towards exhaustivity
 
-mir_build_non_exhaustive_omitted_pattern = some variants are not matched explicitly
-    .help = ensure that all variants are matched explicitly by adding the suggested match arms
-    .note = the matched value is of type `{$scrut_ty}` and the `non_exhaustive_omitted_patterns` attribute was found
-
-mir_build_non_exhaustive_omitted_pattern_lint_on_arm = the lint level must be set on the whole match
-    .help = it no longer has any effect to set the lint level on an individual match arm
-    .label = remove this attribute
-    .suggestion = set the lint level on the whole match
-
 mir_build_non_exhaustive_patterns_type_not_empty = non-exhaustive patterns: type `{$ty}` is non-empty
     .def_note = `{$peeled_ty}` defined here
     .type_note = the matched value is of type `{$ty}`
@@ -260,10 +251,6 @@ mir_build_non_partial_eq_match =
 mir_build_nontrivial_structural_match =
     to use a constant of type `{$non_sm_ty}` in a pattern, the constant's initializer must be trivial or `{$non_sm_ty}` must be annotated with `#[derive(PartialEq, Eq)]`
 
-mir_build_overlapping_range_endpoints = multiple patterns overlap on their endpoints
-    .range = ... with this range
-    .note = you likely meant to write mutually exclusive ranges
-
 mir_build_pattern_not_covered = refutable pattern in {$origin}
     .pattern_ty = the matched value is of type `{$pattern_ty}`
 
@@ -317,13 +304,6 @@ mir_build_unconditional_recursion = function cannot return without recursing
 
 mir_build_unconditional_recursion_call_site_label = recursive call site
 
-mir_build_uncovered = {$count ->
-        [1] pattern `{$witness_1}`
-        [2] patterns `{$witness_1}` and `{$witness_2}`
-        [3] patterns `{$witness_1}`, `{$witness_2}` and `{$witness_3}`
-        *[other] patterns `{$witness_1}`, `{$witness_2}`, `{$witness_3}` and {$remainder} more
-    } not covered
-
 mir_build_union_field_requires_unsafe =
     access to union field is unsafe and requires unsafe block
     .note = the field may not be properly initialized: using uninitialized data will cause undefined behavior
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 8d2a559e73c..1509e2517da 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -1,15 +1,12 @@
-use crate::{
-    fluent_generated as fluent,
-    thir::pattern::{deconstruct_pat::WitnessPat, MatchCheckCtxt},
-};
+use crate::fluent_generated as fluent;
 use rustc_errors::DiagnosticArgValue;
 use rustc_errors::{
     error_code, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
     Handler, IntoDiagnostic, MultiSpan, SubdiagnosticMessage,
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
-use rustc_middle::thir::Pat;
 use rustc_middle::ty::{self, Ty};
+use rustc_pattern_analysis::{errors::Uncovered, usefulness::MatchCheckCtxt};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 
@@ -812,94 +809,6 @@ pub struct NonPartialEqMatch<'tcx> {
     pub non_peq_ty: Ty<'tcx>,
 }
 
-#[derive(LintDiagnostic)]
-#[diag(mir_build_overlapping_range_endpoints)]
-#[note]
-pub struct OverlappingRangeEndpoints<'tcx> {
-    #[label(mir_build_range)]
-    pub range: Span,
-    #[subdiagnostic]
-    pub overlap: Vec<Overlap<'tcx>>,
-}
-
-pub struct Overlap<'tcx> {
-    pub span: Span,
-    pub range: Pat<'tcx>,
-}
-
-impl<'tcx> AddToDiagnostic for Overlap<'tcx> {
-    fn add_to_diagnostic_with<F>(self, diag: &mut Diagnostic, _: F)
-    where
-        F: Fn(&mut Diagnostic, SubdiagnosticMessage) -> SubdiagnosticMessage,
-    {
-        let Overlap { span, range } = self;
-
-        // FIXME(mejrs) unfortunately `#[derive(LintDiagnostic)]`
-        // does not support `#[subdiagnostic(eager)]`...
-        let message = format!("this range overlaps on `{range}`...");
-        diag.span_label(span, message);
-    }
-}
-
-#[derive(LintDiagnostic)]
-#[diag(mir_build_non_exhaustive_omitted_pattern)]
-#[help]
-#[note]
-pub(crate) struct NonExhaustiveOmittedPattern<'tcx> {
-    pub scrut_ty: Ty<'tcx>,
-    #[subdiagnostic]
-    pub uncovered: Uncovered<'tcx>,
-}
-
-#[derive(LintDiagnostic)]
-#[diag(mir_build_non_exhaustive_omitted_pattern_lint_on_arm)]
-#[help]
-pub(crate) struct NonExhaustiveOmittedPatternLintOnArm {
-    #[label]
-    pub lint_span: Span,
-    #[suggestion(code = "#[{lint_level}({lint_name})]\n", applicability = "maybe-incorrect")]
-    pub suggest_lint_on_match: Option<Span>,
-    pub lint_level: &'static str,
-    pub lint_name: &'static str,
-}
-
-#[derive(Subdiagnostic)]
-#[label(mir_build_uncovered)]
-pub(crate) struct Uncovered<'tcx> {
-    #[primary_span]
-    span: Span,
-    count: usize,
-    witness_1: Pat<'tcx>,
-    witness_2: Pat<'tcx>,
-    witness_3: Pat<'tcx>,
-    remainder: usize,
-}
-
-impl<'tcx> Uncovered<'tcx> {
-    pub fn new<'p>(
-        span: Span,
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        witnesses: Vec<WitnessPat<'tcx>>,
-    ) -> Self {
-        let witness_1 = witnesses.get(0).unwrap().to_diagnostic_pat(cx);
-        Self {
-            span,
-            count: witnesses.len(),
-            // Substitute dummy values if witnesses is smaller than 3. These will never be read.
-            witness_2: witnesses
-                .get(1)
-                .map(|w| w.to_diagnostic_pat(cx))
-                .unwrap_or_else(|| witness_1.clone()),
-            witness_3: witnesses
-                .get(2)
-                .map(|w| w.to_diagnostic_pat(cx))
-                .unwrap_or_else(|| witness_1.clone()),
-            witness_1,
-            remainder: witnesses.len().saturating_sub(3),
-        }
-    }
-}
-
 #[derive(Diagnostic)]
 #[diag(mir_build_pattern_not_covered, code = "E0005")]
 pub(crate) struct PatternNotCovered<'s, 'tcx> {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 41510d31530..ca5823a860e 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -1,5 +1,7 @@
-use super::deconstruct_pat::{Constructor, DeconstructedPat, WitnessPat};
-use super::usefulness::{
+use rustc_pattern_analysis::constructor::Constructor;
+use rustc_pattern_analysis::errors::Uncovered;
+use rustc_pattern_analysis::pat::{DeconstructedPat, WitnessPat};
+use rustc_pattern_analysis::usefulness::{
     compute_match_usefulness, MatchArm, MatchCheckCtxt, Usefulness, UsefulnessReport,
 };
 
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
deleted file mode 100644
index ef20b0f039b..00000000000
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ /dev/null
@@ -1,1964 +0,0 @@
-//! As explained in [`super::usefulness`], values and patterns are made from constructors applied to
-//! fields. This file defines a `Constructor` enum, a `Fields` struct, and various operations to
-//! manipulate them and convert them from/to patterns.
-//!
-//! There are two important bits of core logic in this file: constructor inclusion and constructor
-//! splitting. Constructor inclusion, i.e. whether a constructor is included in/covered by another,
-//! is straightforward and defined in [`Constructor::is_covered_by`].
-//!
-//! Constructor splitting is mentioned in [`super::usefulness`] but not detailed. We describe it
-//! precisely here.
-//!
-//!
-//!
-//! # Constructor grouping and splitting
-//!
-//! As explained in the corresponding section in [`super::usefulness`], to make usefulness tractable
-//! we need to group together constructors that have the same effect when they are used to
-//! specialize the matrix.
-//!
-//! Example:
-//! ```compile_fail,E0004
-//! match (0, false) {
-//!     (0 ..=100, true) => {}
-//!     (50..=150, false) => {}
-//!     (0 ..=200, _) => {}
-//! }
-//! ```
-//!
-//! In this example we can restrict specialization to 5 cases: `0..50`, `50..=100`, `101..=150`,
-//! `151..=200` and `200..`.
-//!
-//! In [`super::usefulness`], we had said that `specialize` only takes value-only constructors. We
-//! now relax this restriction: we allow `specialize` to take constructors like `0..50` as long as
-//! we're careful to only do that with constructors that make sense. For example, `specialize(0..50,
-//! (0..=100, true))` is sensible, but `specialize(50..=200, (0..=100, true))` is not.
-//!
-//! Constructor splitting looks at the constructors in the first column of the matrix and constructs
-//! such a sensible set of constructors. Formally, we want to find a smallest disjoint set of
-//! constructors:
-//! - Whose union covers the whole type, and
-//! - That have no non-trivial intersection with any of the constructors in the column (i.e. they're
-//!     each either disjoint with or covered by any given column constructor).
-//!
-//! We compute this in two steps: first [`ConstructorSet::for_ty`] determines the set of all
-//! possible constructors for the type. Then [`ConstructorSet::split`] looks at the column of
-//! constructors and splits the set into groups accordingly. The precise invariants of
-//! [`ConstructorSet::split`] is described in [`SplitConstructorSet`].
-//!
-//! Constructor splitting has two interesting special cases: integer range splitting (see
-//! [`IntRange::split`]) and slice splitting (see [`Slice::split`]).
-//!
-//!
-//!
-//! # The `Missing` constructor
-//!
-//! We detail a special case of constructor splitting that is a bit subtle. Take the following:
-//!
-//! ```
-//! enum Direction { North, South, East, West }
-//! # let wind = (Direction::North, 0u8);
-//! match wind {
-//!     (Direction::North, 50..) => {}
-//!     (_, _) => {}
-//! }
-//! ```
-//!
-//! Here we expect constructor splitting to output two cases: `North`, and "everything else". This
-//! "everything else" is represented by [`Constructor::Missing`]. Unlike other constructors, it's a
-//! bit contextual: to know the exact list of constructors it represents we have to look at the
-//! column. In practice however we don't need to, because by construction it only matches rows that
-//! have wildcards. This is how this constructor is special: the only constructor that covers it is
-//! `Wildcard`.
-//!
-//! The only place where we care about which constructors `Missing` represents is in diagnostics
-//! (see `super::usefulness::WitnessMatrix::apply_constructor`).
-//!
-//! We choose whether to specialize with `Missing` in
-//! `super::usefulness::compute_exhaustiveness_and_reachability`.
-//!
-//!
-//!
-//! ## Empty types, empty constructors, and the `exhaustive_patterns` feature
-//!
-//! An empty type is a type that has no valid value, like `!`, `enum Void {}`, or `Result<!, !>`.
-//! They require careful handling.
-//!
-//! First, for soundness reasons related to the possible existence of invalid values, by default we
-//! don't treat empty types as empty. We force them to be matched with wildcards. Except if the
-//! `exhaustive_patterns` feature is turned on, in which case we do treat them as empty. And also
-//! except if the type has no constructors (like `enum Void {}` but not like `Result<!, !>`), we
-//! specifically allow `match void {}` to be exhaustive. There are additionally considerations of
-//! place validity that are handled in `super::usefulness`. Yes this is a bit tricky.
-//!
-//! The second thing is that regardless of the above, it is always allowed to use all the
-//! constructors of a type. For example, all the following is ok:
-//!
-//! ```rust,ignore(example)
-//! # #![feature(never_type)]
-//! # #![feature(exhaustive_patterns)]
-//! fn foo(x: Option<!>) {
-//!   match x {
-//!     None => {}
-//!     Some(_) => {}
-//!   }
-//! }
-//! fn bar(x: &[!]) -> u32 {
-//!   match x {
-//!     [] => 1,
-//!     [_] => 2,
-//!     [_, _] => 3,
-//!   }
-//! }
-//! ```
-//!
-//! Moreover, take the following:
-//!
-//! ```rust
-//! # #![feature(never_type)]
-//! # #![feature(exhaustive_patterns)]
-//! # let x = None::<!>;
-//! match x {
-//!   None => {}
-//! }
-//! ```
-//!
-//! On a normal type, we would identify `Some` as missing and tell the user. If `x: Option<!>`
-//! however (and `exhaustive_patterns` is on), it's ok to omit `Some`. When listing the constructors
-//! of a type, we must therefore track which can be omitted.
-//!
-//! Let's call "empty" a constructor that matches no valid value for the type, like `Some` for the
-//! type `Option<!>`. What this all means is that `ConstructorSet` must know which constructors are
-//! empty. The difference between empty and nonempty constructors is that empty constructors need
-//! not be present for the match to be exhaustive.
-//!
-//! A final remark: empty constructors of arity 0 break specialization, we must avoid them. The
-//! reason is that if we specialize by them, nothing remains to witness the emptiness; the rest of
-//! the algorithm can't distinguish them from a nonempty constructor. The only known case where this
-//! could happen is the `[..]` pattern on `[!; N]` with `N > 0` so we must take care to not emit it.
-//!
-//! This is all handled by [`ConstructorSet::for_ty`] and [`ConstructorSet::split`]. The invariants
-//! of [`SplitConstructorSet`] are also of interest.
-//!
-//!
-//!
-//! ## Opaque patterns
-//!
-//! Some patterns, such as constants that are not allowed to be matched structurally, cannot be
-//! inspected, which we handle with `Constructor::Opaque`. Since we know nothing of these patterns,
-//! we assume they never cover each other. In order to respect the invariants of
-//! [`SplitConstructorSet`], we give each `Opaque` constructor a unique id so we can recognize it.
-
-use std::cell::Cell;
-use std::cmp::{self, max, min, Ordering};
-use std::fmt;
-use std::iter::once;
-
-use smallvec::{smallvec, SmallVec};
-
-use rustc_apfloat::ieee::{DoubleS, IeeeFloat, SingleS};
-use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fx::FxHashSet;
-use rustc_hir::RangeEnd;
-use rustc_index::{Idx, IndexVec};
-use rustc_middle::middle::stability::EvalResult;
-use rustc_middle::mir;
-use rustc_middle::mir::interpret::Scalar;
-use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange, PatRangeBoundary};
-use rustc_middle::ty::layout::IntegerExt;
-use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
-use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, Integer, VariantIdx, FIRST_VARIANT};
-
-use self::Constructor::*;
-use self::MaybeInfiniteInt::*;
-use self::SliceKind::*;
-
-use super::usefulness::{MatchCheckCtxt, PatCtxt};
-
-/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
-fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
-    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
-        if let PatKind::Or { pats } = &pat.kind {
-            for pat in pats.iter() {
-                expand(pat, vec);
-            }
-        } else {
-            vec.push(pat)
-        }
-    }
-
-    let mut pats = Vec::new();
-    expand(pat, &mut pats);
-    pats
-}
-
-/// Whether we have seen a constructor in the column or not.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-enum Presence {
-    Unseen,
-    Seen,
-}
-
-/// A possibly infinite integer. Values are encoded such that the ordering on `u128` matches the
-/// natural order on the original type. For example, `-128i8` is encoded as `0` and `127i8` as
-/// `255`. See `signed_bias` for details.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub(crate) enum MaybeInfiniteInt {
-    NegInfinity,
-    /// Encoded value. DO NOT CONSTRUCT BY HAND; use `new_finite`.
-    Finite(u128),
-    /// The integer after `u128::MAX`. We need it to represent `x..=u128::MAX` as an exclusive range.
-    JustAfterMax,
-    PosInfinity,
-}
-
-impl MaybeInfiniteInt {
-    // The return value of `signed_bias` should be XORed with a value to encode/decode it.
-    fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
-        match *ty.kind() {
-            ty::Int(ity) => {
-                let bits = Integer::from_int_ty(&tcx, ity).size().bits() as u128;
-                1u128 << (bits - 1)
-            }
-            _ => 0,
-        }
-    }
-
-    fn new_finite(tcx: TyCtxt<'_>, ty: Ty<'_>, bits: u128) -> Self {
-        let bias = Self::signed_bias(tcx, ty);
-        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
-        // type-independent.
-        let x = bits ^ bias;
-        Finite(x)
-    }
-    fn from_pat_range_bdy<'tcx>(
-        bdy: PatRangeBoundary<'tcx>,
-        ty: Ty<'tcx>,
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> Self {
-        match bdy {
-            PatRangeBoundary::NegInfinity => NegInfinity,
-            PatRangeBoundary::Finite(value) => {
-                let bits = value.eval_bits(tcx, param_env);
-                Self::new_finite(tcx, ty, bits)
-            }
-            PatRangeBoundary::PosInfinity => PosInfinity,
-        }
-    }
-
-    /// Used only for diagnostics.
-    /// Note: it is possible to get `isize/usize::MAX+1` here, as explained in the doc for
-    /// [`IntRange::split`]. This cannot be represented as a `Const`, so we represent it with
-    /// `PosInfinity`.
-    fn to_diagnostic_pat_range_bdy<'tcx>(
-        self,
-        ty: Ty<'tcx>,
-        tcx: TyCtxt<'tcx>,
-    ) -> PatRangeBoundary<'tcx> {
-        match self {
-            NegInfinity => PatRangeBoundary::NegInfinity,
-            Finite(x) => {
-                let bias = Self::signed_bias(tcx, ty);
-                let bits = x ^ bias;
-                let size = ty.primitive_size(tcx);
-                match Scalar::try_from_uint(bits, size) {
-                    Some(scalar) => {
-                        let value = mir::Const::from_scalar(tcx, scalar, ty);
-                        PatRangeBoundary::Finite(value)
-                    }
-                    // The value doesn't fit. Since `x >= 0` and 0 always encodes the minimum value
-                    // for a type, the problem isn't that the value is too small. So it must be too
-                    // large.
-                    None => PatRangeBoundary::PosInfinity,
-                }
-            }
-            JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
-        }
-    }
-
-    /// Note: this will not turn a finite value into an infinite one or vice-versa.
-    pub(crate) fn minus_one(self) -> Self {
-        match self {
-            Finite(n) => match n.checked_sub(1) {
-                Some(m) => Finite(m),
-                None => bug!(),
-            },
-            JustAfterMax => Finite(u128::MAX),
-            x => x,
-        }
-    }
-    /// Note: this will not turn a finite value into an infinite one or vice-versa.
-    pub(crate) fn plus_one(self) -> Self {
-        match self {
-            Finite(n) => match n.checked_add(1) {
-                Some(m) => Finite(m),
-                None => JustAfterMax,
-            },
-            JustAfterMax => bug!(),
-            x => x,
-        }
-    }
-}
-
-/// An exclusive interval, used for precise integer exhaustiveness checking. `IntRange`s always
-/// store a contiguous range.
-///
-/// `IntRange` is never used to encode an empty range or a "range" that wraps around the (offset)
-/// space: i.e., `range.lo < range.hi`.
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub(crate) struct IntRange {
-    pub(crate) lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
-    pub(crate) hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
-}
-
-impl IntRange {
-    #[inline]
-    pub(super) fn is_integral(ty: Ty<'_>) -> bool {
-        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_))
-    }
-
-    /// Best effort; will not know that e.g. `255u8..` is a singleton.
-    pub(super) fn is_singleton(&self) -> bool {
-        // Since `lo` and `hi` can't be the same `Infinity` and `plus_one` never changes from finite
-        // to infinite, this correctly only detects ranges that contain exacly one `Finite(x)`.
-        self.lo.plus_one() == self.hi
-    }
-
-    #[inline]
-    fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
-        let x = MaybeInfiniteInt::new_finite(tcx, ty, bits);
-        IntRange { lo: x, hi: x.plus_one() }
-    }
-
-    #[inline]
-    fn from_range(lo: MaybeInfiniteInt, mut hi: MaybeInfiniteInt, end: RangeEnd) -> IntRange {
-        if end == RangeEnd::Included {
-            hi = hi.plus_one();
-        }
-        if lo >= hi {
-            // This should have been caught earlier by E0030.
-            bug!("malformed range pattern: {lo:?}..{hi:?}");
-        }
-        IntRange { lo, hi }
-    }
-
-    fn is_subrange(&self, other: &Self) -> bool {
-        other.lo <= self.lo && self.hi <= other.hi
-    }
-
-    fn intersection(&self, other: &Self) -> Option<Self> {
-        if self.lo < other.hi && other.lo < self.hi {
-            Some(IntRange { lo: max(self.lo, other.lo), hi: min(self.hi, other.hi) })
-        } else {
-            None
-        }
-    }
-
-    /// Partition a range of integers into disjoint subranges. This does constructor splitting for
-    /// integer ranges as explained at the top of the file.
-    ///
-    /// This returns an output that covers `self`. The output is split so that the only
-    /// intersections between an output range and a column range are inclusions. No output range
-    /// straddles the boundary of one of the inputs.
-    ///
-    /// Additionally, we track for each output range whether it is covered by one of the column ranges or not.
-    ///
-    /// The following input:
-    /// ```text
-    ///   (--------------------------) // `self`
-    /// (------) (----------)    (-)
-    ///     (------) (--------)
-    /// ```
-    /// is first intersected with `self`:
-    /// ```text
-    ///   (--------------------------) // `self`
-    ///   (----) (----------)    (-)
-    ///     (------) (--------)
-    /// ```
-    /// and then iterated over as follows:
-    /// ```text
-    ///   (-(--)-(-)-(------)-)--(-)-
-    /// ```
-    /// where each sequence of dashes is an output range, and dashes outside parentheses are marked
-    /// as `Presence::Missing`.
-    ///
-    /// ## `isize`/`usize`
-    ///
-    /// Whereas a wildcard of type `i32` stands for the range `i32::MIN..=i32::MAX`, a `usize`
-    /// wildcard stands for `0..PosInfinity` and a `isize` wildcard stands for
-    /// `NegInfinity..PosInfinity`. In other words, as far as `IntRange` is concerned, there are
-    /// values before `isize::MIN` and after `usize::MAX`/`isize::MAX`.
-    /// This is to avoid e.g. `0..(u32::MAX as usize)` from being exhaustive on one architecture and
-    /// not others. This was decided in <https://github.com/rust-lang/rfcs/pull/2591>.
-    ///
-    /// These infinities affect splitting subtly: it is possible to get `NegInfinity..0` and
-    /// `usize::MAX+1..PosInfinity` in the output. Diagnostics must be careful to handle these
-    /// fictitious ranges sensibly.
-    fn split(
-        &self,
-        column_ranges: impl Iterator<Item = IntRange>,
-    ) -> impl Iterator<Item = (Presence, IntRange)> {
-        // The boundaries of ranges in `column_ranges` intersected with `self`.
-        // We do parenthesis matching for input ranges. A boundary counts as +1 if it starts
-        // a range and -1 if it ends it. When the count is > 0 between two boundaries, we
-        // are within an input range.
-        let mut boundaries: Vec<(MaybeInfiniteInt, isize)> = column_ranges
-            .filter_map(|r| self.intersection(&r))
-            .flat_map(|r| [(r.lo, 1), (r.hi, -1)])
-            .collect();
-        // We sort by boundary, and for each boundary we sort the "closing parentheses" first. The
-        // order of +1/-1 for a same boundary value is actually irrelevant, because we only look at
-        // the accumulated count between distinct boundary values.
-        boundaries.sort_unstable();
-
-        // Accumulate parenthesis counts.
-        let mut paren_counter = 0isize;
-        // Gather pairs of adjacent boundaries.
-        let mut prev_bdy = self.lo;
-        boundaries
-            .into_iter()
-            // End with the end of the range. The count is ignored.
-            .chain(once((self.hi, 0)))
-            // List pairs of adjacent boundaries and the count between them.
-            .map(move |(bdy, delta)| {
-                // `delta` affects the count as we cross `bdy`, so the relevant count between
-                // `prev_bdy` and `bdy` is untouched by `delta`.
-                let ret = (prev_bdy, paren_counter, bdy);
-                prev_bdy = bdy;
-                paren_counter += delta;
-                ret
-            })
-            // Skip empty ranges.
-            .filter(|&(prev_bdy, _, bdy)| prev_bdy != bdy)
-            // Convert back to ranges.
-            .map(move |(prev_bdy, paren_count, bdy)| {
-                use Presence::*;
-                let presence = if paren_count > 0 { Seen } else { Unseen };
-                let range = IntRange { lo: prev_bdy, hi: bdy };
-                (presence, range)
-            })
-    }
-
-    /// Whether the range denotes the fictitious values before `isize::MIN` or after
-    /// `usize::MAX`/`isize::MAX` (see doc of [`IntRange::split`] for why these exist).
-    pub(crate) fn is_beyond_boundaries<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool {
-        ty.is_ptr_sized_integral() && {
-            // The two invalid ranges are `NegInfinity..isize::MIN` (represented as
-            // `NegInfinity..0`), and `{u,i}size::MAX+1..PosInfinity`. `to_diagnostic_pat_range_bdy`
-            // converts `MAX+1` to `PosInfinity`, and we couldn't have `PosInfinity` in `self.lo`
-            // otherwise.
-            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            matches!(lo, PatRangeBoundary::PosInfinity)
-                || matches!(self.hi, MaybeInfiniteInt::Finite(0))
-        }
-    }
-    /// Only used for displaying the range.
-    pub(super) fn to_diagnostic_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
-        let kind = if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
-            PatKind::Wild
-        } else if self.is_singleton() {
-            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            let value = lo.as_finite().unwrap();
-            PatKind::Constant { value }
-        } else {
-            // We convert to an inclusive range for diagnostics.
-            let mut end = RangeEnd::Included;
-            let mut lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            if matches!(lo, PatRangeBoundary::PosInfinity) {
-                // The only reason to get `PosInfinity` here is the special case where
-                // `to_diagnostic_pat_range_bdy` found `{u,i}size::MAX+1`. So the range denotes the
-                // fictitious values after `{u,i}size::MAX` (see [`IntRange::split`] for why we do
-                // this). We show this to the user as `usize::MAX..` which is slightly incorrect but
-                // probably clear enough.
-                let c = ty.numeric_max_val(tcx).unwrap();
-                let value = mir::Const::from_ty_const(c, tcx);
-                lo = PatRangeBoundary::Finite(value);
-            }
-            let hi = if matches!(self.hi, MaybeInfiniteInt::Finite(0)) {
-                // The range encodes `..ty::MIN`, so we can't convert it to an inclusive range.
-                end = RangeEnd::Excluded;
-                self.hi
-            } else {
-                self.hi.minus_one()
-            };
-            let hi = hi.to_diagnostic_pat_range_bdy(ty, tcx);
-            PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
-        };
-
-        Pat { ty, span: DUMMY_SP, kind }
-    }
-}
-
-/// Note: this will render signed ranges incorrectly. To render properly, convert to a pattern
-/// first.
-impl fmt::Debug for IntRange {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Finite(lo) = self.lo {
-            write!(f, "{lo}")?;
-        }
-        write!(f, "{}", RangeEnd::Excluded)?;
-        if let Finite(hi) = self.hi {
-            write!(f, "{hi}")?;
-        }
-        Ok(())
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-enum SliceKind {
-    /// Patterns of length `n` (`[x, y]`).
-    FixedLen(usize),
-    /// Patterns using the `..` notation (`[x, .., y]`).
-    /// Captures any array constructor of `length >= i + j`.
-    /// In the case where `array_len` is `Some(_)`,
-    /// this indicates that we only care about the first `i` and the last `j` values of the array,
-    /// and everything in between is a wildcard `_`.
-    VarLen(usize, usize),
-}
-
-impl SliceKind {
-    fn arity(self) -> usize {
-        match self {
-            FixedLen(length) => length,
-            VarLen(prefix, suffix) => prefix + suffix,
-        }
-    }
-
-    /// Whether this pattern includes patterns of length `other_len`.
-    fn covers_length(self, other_len: usize) -> bool {
-        match self {
-            FixedLen(len) => len == other_len,
-            VarLen(prefix, suffix) => prefix + suffix <= other_len,
-        }
-    }
-}
-
-/// A constructor for array and slice patterns.
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub(super) struct Slice {
-    /// `None` if the matched value is a slice, `Some(n)` if it is an array of size `n`.
-    array_len: Option<usize>,
-    /// The kind of pattern it is: fixed-length `[x, y]` or variable length `[x, .., y]`.
-    kind: SliceKind,
-}
-
-impl Slice {
-    fn new(array_len: Option<usize>, kind: SliceKind) -> Self {
-        let kind = match (array_len, kind) {
-            // If the middle `..` has length 0, we effectively have a fixed-length pattern.
-            (Some(len), VarLen(prefix, suffix)) if prefix + suffix == len => FixedLen(len),
-            (Some(len), VarLen(prefix, suffix)) if prefix + suffix > len => bug!(
-                "Slice pattern of length {} longer than its array length {len}",
-                prefix + suffix
-            ),
-            _ => kind,
-        };
-        Slice { array_len, kind }
-    }
-
-    fn arity(self) -> usize {
-        self.kind.arity()
-    }
-
-    /// See `Constructor::is_covered_by`
-    fn is_covered_by(self, other: Self) -> bool {
-        other.kind.covers_length(self.arity())
-    }
-
-    /// This computes constructor splitting for variable-length slices, as explained at the top of
-    /// the file.
-    ///
-    /// A slice pattern `[x, .., y]` behaves like the infinite or-pattern `[x, y] | [x, _, y] | [x,
-    /// _, _, y] | etc`. The corresponding value constructors are fixed-length array constructors of
-    /// corresponding lengths. We obviously can't list this infinitude of constructors.
-    /// Thankfully, it turns out that for each finite set of slice patterns, all sufficiently large
-    /// array lengths are equivalent.
-    ///
-    /// Let's look at an example, where we are trying to split the last pattern:
-    /// ```
-    /// # fn foo(x: &[bool]) {
-    /// match x {
-    ///     [true, true, ..] => {}
-    ///     [.., false, false] => {}
-    ///     [..] => {}
-    /// }
-    /// # }
-    /// ```
-    /// Here are the results of specialization for the first few lengths:
-    /// ```
-    /// # fn foo(x: &[bool]) { match x {
-    /// // length 0
-    /// [] => {}
-    /// // length 1
-    /// [_] => {}
-    /// // length 2
-    /// [true, true] => {}
-    /// [false, false] => {}
-    /// [_, _] => {}
-    /// // length 3
-    /// [true, true,  _    ] => {}
-    /// [_,    false, false] => {}
-    /// [_,    _,     _    ] => {}
-    /// // length 4
-    /// [true, true, _,     _    ] => {}
-    /// [_,    _,    false, false] => {}
-    /// [_,    _,    _,     _    ] => {}
-    /// // length 5
-    /// [true, true, _, _,     _    ] => {}
-    /// [_,    _,    _, false, false] => {}
-    /// [_,    _,    _, _,     _    ] => {}
-    /// # _ => {}
-    /// # }}
-    /// ```
-    ///
-    /// We see that above length 4, we are simply inserting columns full of wildcards in the middle.
-    /// This means that specialization and witness computation with slices of length `l >= 4` will
-    /// give equivalent results regardless of `l`. This applies to any set of slice patterns: there
-    /// will be a length `L` above which all lengths behave the same. This is exactly what we need
-    /// for constructor splitting.
-    ///
-    /// A variable-length slice pattern covers all lengths from its arity up to infinity. As we just
-    /// saw, we can split this in two: lengths below `L` are treated individually with a
-    /// fixed-length slice each; lengths above `L` are grouped into a single variable-length slice
-    /// constructor.
-    ///
-    /// For each variable-length slice pattern `p` with a prefix of length `plₚ` and suffix of
-    /// length `slₚ`, only the first `plₚ` and the last `slₚ` elements are examined. Therefore, as
-    /// long as `L` is positive (to avoid concerns about empty types), all elements after the
-    /// maximum prefix length and before the maximum suffix length are not examined by any
-    /// variable-length pattern, and therefore can be ignored. This gives us a way to compute `L`.
-    ///
-    /// Additionally, if fixed-length patterns exist, we must pick an `L` large enough to miss them,
-    /// so we can pick `L = max(max(FIXED_LEN)+1, max(PREFIX_LEN) + max(SUFFIX_LEN))`.
-    /// `max_slice` below will be made to have this arity `L`.
-    ///
-    /// If `self` is fixed-length, it is returned as-is.
-    ///
-    /// Additionally, we track for each output slice whether it is covered by one of the column slices or not.
-    fn split(
-        self,
-        column_slices: impl Iterator<Item = Slice>,
-    ) -> impl Iterator<Item = (Presence, Slice)> {
-        // Range of lengths below `L`.
-        let smaller_lengths;
-        let arity = self.arity();
-        let mut max_slice = self.kind;
-        // Tracks the smallest variable-length slice we've seen. Any slice arity above it is
-        // therefore `Presence::Seen` in the column.
-        let mut min_var_len = usize::MAX;
-        // Tracks the fixed-length slices we've seen, to mark them as `Presence::Seen`.
-        let mut seen_fixed_lens = FxHashSet::default();
-        match &mut max_slice {
-            VarLen(max_prefix_len, max_suffix_len) => {
-                // A length larger than any fixed-length slice encountered.
-                // We start at 1 in case the subtype is empty because in that case the zero-length
-                // slice must be treated separately from the rest.
-                let mut fixed_len_upper_bound = 1;
-                // We grow `max_slice` to be larger than all slices encountered, as described above.
-                // `L` is `max_slice.arity()`. For diagnostics, we keep the prefix and suffix
-                // lengths separate.
-                for slice in column_slices {
-                    match slice.kind {
-                        FixedLen(len) => {
-                            fixed_len_upper_bound = cmp::max(fixed_len_upper_bound, len + 1);
-                            seen_fixed_lens.insert(len);
-                        }
-                        VarLen(prefix, suffix) => {
-                            *max_prefix_len = cmp::max(*max_prefix_len, prefix);
-                            *max_suffix_len = cmp::max(*max_suffix_len, suffix);
-                            min_var_len = cmp::min(min_var_len, prefix + suffix);
-                        }
-                    }
-                }
-                // If `fixed_len_upper_bound >= L`, we set `L` to `fixed_len_upper_bound`.
-                if let Some(delta) =
-                    fixed_len_upper_bound.checked_sub(*max_prefix_len + *max_suffix_len)
-                {
-                    *max_prefix_len += delta
-                }
-
-                // We cap the arity of `max_slice` at the array size.
-                match self.array_len {
-                    Some(len) if max_slice.arity() >= len => max_slice = FixedLen(len),
-                    _ => {}
-                }
-
-                smaller_lengths = match self.array_len {
-                    // The only admissible fixed-length slice is one of the array size. Whether `max_slice`
-                    // is fixed-length or variable-length, it will be the only relevant slice to output
-                    // here.
-                    Some(_) => 0..0, // empty range
-                    // We need to cover all arities in the range `(arity..infinity)`. We split that
-                    // range into two: lengths smaller than `max_slice.arity()` are treated
-                    // independently as fixed-lengths slices, and lengths above are captured by
-                    // `max_slice`.
-                    None => self.arity()..max_slice.arity(),
-                };
-            }
-            FixedLen(_) => {
-                // No need to split here. We only track presence.
-                for slice in column_slices {
-                    match slice.kind {
-                        FixedLen(len) => {
-                            if len == arity {
-                                seen_fixed_lens.insert(len);
-                            }
-                        }
-                        VarLen(prefix, suffix) => {
-                            min_var_len = cmp::min(min_var_len, prefix + suffix);
-                        }
-                    }
-                }
-                smaller_lengths = 0..0;
-            }
-        };
-
-        smaller_lengths.map(FixedLen).chain(once(max_slice)).map(move |kind| {
-            let arity = kind.arity();
-            let seen = if min_var_len <= arity || seen_fixed_lens.contains(&arity) {
-                Presence::Seen
-            } else {
-                Presence::Unseen
-            };
-            (seen, Slice::new(self.array_len, kind))
-        })
-    }
-}
-
-/// A globally unique id to distinguish `Opaque` patterns.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub(super) struct OpaqueId(u32);
-
-impl OpaqueId {
-    fn new() -> Self {
-        use std::sync::atomic::{AtomicU32, Ordering};
-        static OPAQUE_ID: AtomicU32 = AtomicU32::new(0);
-        OpaqueId(OPAQUE_ID.fetch_add(1, Ordering::SeqCst))
-    }
-}
-
-/// A value can be decomposed into a constructor applied to some fields. This struct represents
-/// the constructor. See also `Fields`.
-///
-/// `pat_constructor` retrieves the constructor corresponding to a pattern.
-/// `specialize_constructor` returns the list of fields corresponding to a pattern, given a
-/// constructor. `Constructor::apply` reconstructs the pattern from a pair of `Constructor` and
-/// `Fields`.
-#[derive(Clone, Debug, PartialEq)]
-pub(super) enum Constructor<'tcx> {
-    /// The constructor for patterns that have a single constructor, like tuples, struct patterns,
-    /// and references. Fixed-length arrays are treated separately with `Slice`.
-    Single,
-    /// Enum variants.
-    Variant(VariantIdx),
-    /// Booleans
-    Bool(bool),
-    /// Ranges of integer literal values (`2`, `2..=5` or `2..5`).
-    IntRange(IntRange),
-    /// Ranges of floating-point literal values (`2.0..=5.2`).
-    F32Range(IeeeFloat<SingleS>, IeeeFloat<SingleS>, RangeEnd),
-    F64Range(IeeeFloat<DoubleS>, IeeeFloat<DoubleS>, RangeEnd),
-    /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately.
-    Str(mir::Const<'tcx>),
-    /// Array and slice patterns.
-    Slice(Slice),
-    /// Constants that must not be matched structurally. They are treated as black boxes for the
-    /// purposes of exhaustiveness: we must not inspect them, and they don't count towards making a
-    /// match exhaustive.
-    /// Carries an id that must be unique within a match. We need this to ensure the invariants of
-    /// [`SplitConstructorSet`].
-    Opaque(OpaqueId),
-    /// Or-pattern.
-    Or,
-    /// Wildcard pattern.
-    Wildcard,
-    /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. Also used
-    /// for those types for which we cannot list constructors explicitly, like `f64` and `str`.
-    NonExhaustive,
-    /// Fake extra constructor for variants that should not be mentioned in diagnostics.
-    /// We use this for variants behind an unstable gate as well as
-    /// `#[doc(hidden)]` ones.
-    Hidden,
-    /// Fake extra constructor for constructors that are not seen in the matrix, as explained at the
-    /// top of the file.
-    Missing,
-}
-
-impl<'tcx> Constructor<'tcx> {
-    pub(super) fn is_non_exhaustive(&self) -> bool {
-        matches!(self, NonExhaustive)
-    }
-
-    pub(super) fn as_variant(&self) -> Option<VariantIdx> {
-        match self {
-            Variant(i) => Some(*i),
-            _ => None,
-        }
-    }
-    fn as_bool(&self) -> Option<bool> {
-        match self {
-            Bool(b) => Some(*b),
-            _ => None,
-        }
-    }
-    pub(super) fn as_int_range(&self) -> Option<&IntRange> {
-        match self {
-            IntRange(range) => Some(range),
-            _ => None,
-        }
-    }
-    fn as_slice(&self) -> Option<Slice> {
-        match self {
-            Slice(slice) => Some(*slice),
-            _ => None,
-        }
-    }
-
-    fn variant_index_for_adt(&self, adt: ty::AdtDef<'tcx>) -> VariantIdx {
-        match *self {
-            Variant(idx) => idx,
-            Single => {
-                assert!(!adt.is_enum());
-                FIRST_VARIANT
-            }
-            _ => bug!("bad constructor {:?} for adt {:?}", self, adt),
-        }
-    }
-
-    /// The number of fields for this constructor. This must be kept in sync with
-    /// `Fields::wildcards`.
-    pub(super) fn arity(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> usize {
-        match self {
-            Single | Variant(_) => match pcx.ty.kind() {
-                ty::Tuple(fs) => fs.len(),
-                ty::Ref(..) => 1,
-                ty::Adt(adt, ..) => {
-                    if adt.is_box() {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        1
-                    } else {
-                        let variant = &adt.variant(self.variant_index_for_adt(*adt));
-                        Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant).count()
-                    }
-                }
-                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx.ty),
-            },
-            Slice(slice) => slice.arity(),
-            Bool(..)
-            | IntRange(..)
-            | F32Range(..)
-            | F64Range(..)
-            | Str(..)
-            | Opaque(..)
-            | NonExhaustive
-            | Hidden
-            | Missing { .. }
-            | Wildcard => 0,
-            Or => bug!("The `Or` constructor doesn't have a fixed arity"),
-        }
-    }
-
-    /// Returns whether `self` is covered by `other`, i.e. whether `self` is a subset of `other`.
-    /// For the simple cases, this is simply checking for equality. For the "grouped" constructors,
-    /// this checks for inclusion.
-    // We inline because this has a single call site in `Matrix::specialize_constructor`.
-    #[inline]
-    pub(super) fn is_covered_by<'p>(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, other: &Self) -> bool {
-        match (self, other) {
-            (Wildcard, _) => {
-                span_bug!(
-                    pcx.cx.scrut_span,
-                    "Constructor splitting should not have returned `Wildcard`"
-                )
-            }
-            // Wildcards cover anything
-            (_, Wildcard) => true,
-            // Only a wildcard pattern can match these special constructors.
-            (Missing { .. } | NonExhaustive | Hidden, _) => false,
-
-            (Single, Single) => true,
-            (Variant(self_id), Variant(other_id)) => self_id == other_id,
-            (Bool(self_b), Bool(other_b)) => self_b == other_b,
-
-            (IntRange(self_range), IntRange(other_range)) => self_range.is_subrange(other_range),
-            (F32Range(self_from, self_to, self_end), F32Range(other_from, other_to, other_end)) => {
-                self_from.ge(other_from)
-                    && match self_to.partial_cmp(other_to) {
-                        Some(Ordering::Less) => true,
-                        Some(Ordering::Equal) => other_end == self_end,
-                        _ => false,
-                    }
-            }
-            (F64Range(self_from, self_to, self_end), F64Range(other_from, other_to, other_end)) => {
-                self_from.ge(other_from)
-                    && match self_to.partial_cmp(other_to) {
-                        Some(Ordering::Less) => true,
-                        Some(Ordering::Equal) => other_end == self_end,
-                        _ => false,
-                    }
-            }
-            (Str(self_val), Str(other_val)) => {
-                // FIXME Once valtrees are available we can directly use the bytes
-                // in the `Str` variant of the valtree for the comparison here.
-                self_val == other_val
-            }
-            (Slice(self_slice), Slice(other_slice)) => self_slice.is_covered_by(*other_slice),
-
-            // Opaque constructors don't interact with anything unless they come from the
-            // syntactically identical pattern.
-            (Opaque(self_id), Opaque(other_id)) => self_id == other_id,
-            (Opaque(..), _) | (_, Opaque(..)) => false,
-
-            _ => span_bug!(
-                pcx.cx.scrut_span,
-                "trying to compare incompatible constructors {:?} and {:?}",
-                self,
-                other
-            ),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub(super) enum VariantVisibility {
-    /// Variant that doesn't fit the other cases, i.e. most variants.
-    Visible,
-    /// Variant behind an unstable gate or with the `#[doc(hidden)]` attribute. It will not be
-    /// mentioned in diagnostics unless the user mentioned it first.
-    Hidden,
-    /// Variant that matches no value. E.g. `Some::<Option<!>>` if the `exhaustive_patterns` feature
-    /// is enabled. Like `Hidden`, it will not be mentioned in diagnostics unless the user mentioned
-    /// it first.
-    Empty,
-}
-
-/// Describes the set of all constructors for a type. For details, in particular about the emptiness
-/// of constructors, see the top of the file.
-///
-/// In terms of division of responsibility, [`ConstructorSet::split`] handles all of the
-/// `exhaustive_patterns` feature.
-#[derive(Debug)]
-pub(super) enum ConstructorSet {
-    /// The type has a single constructor, e.g. `&T` or a struct. `empty` tracks whether the
-    /// constructor is empty.
-    Single { empty: bool },
-    /// This type has the following list of constructors. If `variants` is empty and
-    /// `non_exhaustive` is false, don't use this; use `NoConstructors` instead.
-    Variants { variants: IndexVec<VariantIdx, VariantVisibility>, non_exhaustive: bool },
-    /// Booleans.
-    Bool,
-    /// The type is spanned by integer values. The range or ranges give the set of allowed values.
-    /// The second range is only useful for `char`.
-    Integers { range_1: IntRange, range_2: Option<IntRange> },
-    /// The type is matched by slices. `array_len` is the compile-time length of the array, if
-    /// known. If `subtype_is_empty`, all constructors are empty except possibly the zero-length
-    /// slice `[]`.
-    Slice { array_len: Option<usize>, subtype_is_empty: bool },
-    /// The constructors cannot be listed, and the type cannot be matched exhaustively. E.g. `str`,
-    /// floats.
-    Unlistable,
-    /// The type has no constructors (not even empty ones). This is `!` and empty enums.
-    NoConstructors,
-}
-
-/// Describes the result of analyzing the constructors in a column of a match.
-///
-/// `present` is morally the set of constructors present in the column, and `missing` is the set of
-/// constructors that exist in the type but are not present in the column.
-///
-/// More formally, if we discard wildcards from the column, this respects the following constraints:
-/// 1. the union of `present`, `missing` and `missing_empty` covers all the constructors of the type
-/// 2. each constructor in `present` is covered by something in the column
-/// 3. no constructor in `missing` or `missing_empty` is covered by anything in the column
-/// 4. each constructor in the column is equal to the union of one or more constructors in `present`
-/// 5. `missing` does not contain empty constructors (see discussion about emptiness at the top of
-///    the file);
-/// 6. `missing_empty` contains only empty constructors
-/// 7. constructors in `present`, `missing` and `missing_empty` are split for the column; in other
-///    words, they are either fully included in or fully disjoint from each constructor in the
-///    column. In yet other words, there are no non-trivial intersections like between `0..10` and
-///    `5..15`.
-///
-/// We must be particularly careful with weird constructors like `Opaque`: they're not formally part
-/// of the `ConstructorSet` for the type, yet if we forgot to include them in `present` we would be
-/// ignoring any row with `Opaque`s in the algorithm. Hence the importance of point 4.
-#[derive(Debug)]
-pub(super) struct SplitConstructorSet<'tcx> {
-    pub(super) present: SmallVec<[Constructor<'tcx>; 1]>,
-    pub(super) missing: Vec<Constructor<'tcx>>,
-    pub(super) missing_empty: Vec<Constructor<'tcx>>,
-}
-
-impl ConstructorSet {
-    /// Creates a set that represents all the constructors of `ty`.
-    ///
-    /// See at the top of the file for considerations of emptiness.
-    #[instrument(level = "debug", skip(cx), ret)]
-    pub(super) fn for_ty<'p, 'tcx>(cx: &MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
-        let make_range = |start, end| {
-            IntRange::from_range(
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, start),
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, end),
-                RangeEnd::Included,
-            )
-        };
-        // This determines the set of all possible constructors for the type `ty`. For numbers,
-        // arrays and slices we use ranges and variable-length slices when appropriate.
-        match ty.kind() {
-            ty::Bool => Self::Bool,
-            ty::Char => {
-                // The valid Unicode Scalar Value ranges.
-                Self::Integers {
-                    range_1: make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
-                    range_2: Some(make_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
-                }
-            }
-            &ty::Int(ity) => {
-                let range = if ty.is_ptr_sized_integral() {
-                    // The min/max values of `isize` are not allowed to be observed.
-                    IntRange { lo: NegInfinity, hi: PosInfinity }
-                } else {
-                    let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
-                    let min = 1u128 << (bits - 1);
-                    let max = min - 1;
-                    make_range(min, max)
-                };
-                Self::Integers { range_1: range, range_2: None }
-            }
-            &ty::Uint(uty) => {
-                let range = if ty.is_ptr_sized_integral() {
-                    // The max value of `usize` is not allowed to be observed.
-                    let lo = MaybeInfiniteInt::new_finite(cx.tcx, ty, 0);
-                    IntRange { lo, hi: PosInfinity }
-                } else {
-                    let size = Integer::from_uint_ty(&cx.tcx, uty).size();
-                    let max = size.truncate(u128::MAX);
-                    make_range(0, max)
-                };
-                Self::Integers { range_1: range, range_2: None }
-            }
-            ty::Slice(sub_ty) => {
-                Self::Slice { array_len: None, subtype_is_empty: cx.is_uninhabited(*sub_ty) }
-            }
-            ty::Array(sub_ty, len) => {
-                // We treat arrays of a constant but unknown length like slices.
-                Self::Slice {
-                    array_len: len.try_eval_target_usize(cx.tcx, cx.param_env).map(|l| l as usize),
-                    subtype_is_empty: cx.is_uninhabited(*sub_ty),
-                }
-            }
-            ty::Adt(def, args) if def.is_enum() => {
-                let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(ty);
-                if def.variants().is_empty() && !is_declared_nonexhaustive {
-                    Self::NoConstructors
-                } else {
-                    let mut variants =
-                        IndexVec::from_elem(VariantVisibility::Visible, def.variants());
-                    for (idx, v) in def.variants().iter_enumerated() {
-                        let variant_def_id = def.variant(idx).def_id;
-                        // Visibly uninhabited variants.
-                        let is_inhabited = v
-                            .inhabited_predicate(cx.tcx, *def)
-                            .instantiate(cx.tcx, args)
-                            .apply(cx.tcx, cx.param_env, cx.module);
-                        // Variants that depend on a disabled unstable feature.
-                        let is_unstable = matches!(
-                            cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
-                            EvalResult::Deny { .. }
-                        );
-                        // Foreign `#[doc(hidden)]` variants.
-                        let is_doc_hidden =
-                            cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
-                        let visibility = if !is_inhabited {
-                            // FIXME: handle empty+hidden
-                            VariantVisibility::Empty
-                        } else if is_unstable || is_doc_hidden {
-                            VariantVisibility::Hidden
-                        } else {
-                            VariantVisibility::Visible
-                        };
-                        variants[idx] = visibility;
-                    }
-
-                    Self::Variants { variants, non_exhaustive: is_declared_nonexhaustive }
-                }
-            }
-            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => {
-                Self::Single { empty: cx.is_uninhabited(ty) }
-            }
-            ty::Never => Self::NoConstructors,
-            // This type is one for which we cannot list constructors, like `str` or `f64`.
-            // FIXME(Nadrieril): which of these are actually allowed?
-            ty::Float(_)
-            | ty::Str
-            | ty::Foreign(_)
-            | ty::RawPtr(_)
-            | ty::FnDef(_, _)
-            | ty::FnPtr(_)
-            | ty::Dynamic(_, _, _)
-            | ty::Closure(_, _)
-            | ty::Coroutine(_, _, _)
-            | ty::Alias(_, _)
-            | ty::Param(_)
-            | ty::Error(_) => Self::Unlistable,
-            ty::CoroutineWitness(_, _) | ty::Bound(_, _) | ty::Placeholder(_) | ty::Infer(_) => {
-                bug!("Encountered unexpected type in `ConstructorSet::for_ty`: {ty:?}")
-            }
-        }
-    }
-
-    /// This analyzes a column of constructors to 1/ determine which constructors of the type (if
-    /// any) are missing; 2/ split constructors to handle non-trivial intersections e.g. on ranges
-    /// or slices. This can get subtle; see [`SplitConstructorSet`] for details of this operation
-    /// and its invariants.
-    #[instrument(level = "debug", skip(self, pcx, ctors), ret)]
-    pub(super) fn split<'a, 'tcx>(
-        &self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
-    ) -> SplitConstructorSet<'tcx>
-    where
-        'tcx: 'a,
-    {
-        let mut present: SmallVec<[_; 1]> = SmallVec::new();
-        // Empty constructors found missing.
-        let mut missing_empty = Vec::new();
-        // Nonempty constructors found missing.
-        let mut missing = Vec::new();
-        // Constructors in `ctors`, except wildcards and opaques.
-        let mut seen = Vec::new();
-        for ctor in ctors.cloned() {
-            match ctor {
-                Opaque(..) => present.push(ctor),
-                Wildcard => {} // discard wildcards
-                _ => seen.push(ctor),
-            }
-        }
-
-        match self {
-            ConstructorSet::Single { empty } => {
-                if !seen.is_empty() {
-                    present.push(Single);
-                } else if *empty {
-                    missing_empty.push(Single);
-                } else {
-                    missing.push(Single);
-                }
-            }
-            ConstructorSet::Variants { variants, non_exhaustive } => {
-                let seen_set: FxHashSet<_> = seen.iter().map(|c| c.as_variant().unwrap()).collect();
-                let mut skipped_a_hidden_variant = false;
-
-                for (idx, visibility) in variants.iter_enumerated() {
-                    let ctor = Variant(idx);
-                    if seen_set.contains(&idx) {
-                        present.push(ctor);
-                    } else {
-                        // We only put visible variants directly into `missing`.
-                        match visibility {
-                            VariantVisibility::Visible => missing.push(ctor),
-                            VariantVisibility::Hidden => skipped_a_hidden_variant = true,
-                            VariantVisibility::Empty => missing_empty.push(ctor),
-                        }
-                    }
-                }
-
-                if skipped_a_hidden_variant {
-                    missing.push(Hidden);
-                }
-                if *non_exhaustive {
-                    missing.push(NonExhaustive);
-                }
-            }
-            ConstructorSet::Bool => {
-                let mut seen_false = false;
-                let mut seen_true = false;
-                for b in seen.iter().map(|ctor| ctor.as_bool().unwrap()) {
-                    if b {
-                        seen_true = true;
-                    } else {
-                        seen_false = true;
-                    }
-                }
-                if seen_false {
-                    present.push(Bool(false));
-                } else {
-                    missing.push(Bool(false));
-                }
-                if seen_true {
-                    present.push(Bool(true));
-                } else {
-                    missing.push(Bool(true));
-                }
-            }
-            ConstructorSet::Integers { range_1, range_2 } => {
-                let seen_ranges: Vec<_> =
-                    seen.iter().map(|ctor| ctor.as_int_range().unwrap().clone()).collect();
-                for (seen, splitted_range) in range_1.split(seen_ranges.iter().cloned()) {
-                    match seen {
-                        Presence::Unseen => missing.push(IntRange(splitted_range)),
-                        Presence::Seen => present.push(IntRange(splitted_range)),
-                    }
-                }
-                if let Some(range_2) = range_2 {
-                    for (seen, splitted_range) in range_2.split(seen_ranges.into_iter()) {
-                        match seen {
-                            Presence::Unseen => missing.push(IntRange(splitted_range)),
-                            Presence::Seen => present.push(IntRange(splitted_range)),
-                        }
-                    }
-                }
-            }
-            ConstructorSet::Slice { array_len, subtype_is_empty } => {
-                let seen_slices = seen.iter().map(|c| c.as_slice().unwrap());
-                let base_slice = Slice::new(*array_len, VarLen(0, 0));
-                for (seen, splitted_slice) in base_slice.split(seen_slices) {
-                    let ctor = Slice(splitted_slice);
-                    match seen {
-                        Presence::Seen => present.push(ctor),
-                        Presence::Unseen => {
-                            if *subtype_is_empty && splitted_slice.arity() != 0 {
-                                // We have subpatterns of an empty type, so the constructor is
-                                // empty.
-                                missing_empty.push(ctor);
-                            } else {
-                                missing.push(ctor);
-                            }
-                        }
-                    }
-                }
-            }
-            ConstructorSet::Unlistable => {
-                // Since we can't list constructors, we take the ones in the column. This might list
-                // some constructors several times but there's not much we can do.
-                present.extend(seen);
-                missing.push(NonExhaustive);
-            }
-            ConstructorSet::NoConstructors => {
-                // In a `MaybeInvalid` place even an empty pattern may be reachable. We therefore
-                // add a dummy empty constructor here, which will be ignored if the place is
-                // `ValidOnly`.
-                missing_empty.push(NonExhaustive);
-            }
-        }
-
-        // We have now grouped all the constructors into 3 buckets: present, missing, missing_empty.
-        // In the absence of the `exhaustive_patterns` feature however, we don't count nested empty
-        // types as empty. Only non-nested `!` or `enum Foo {}` are considered empty.
-        if !pcx.cx.tcx.features().exhaustive_patterns
-            && !(pcx.is_top_level && matches!(self, Self::NoConstructors))
-        {
-            // Treat all missing constructors as nonempty.
-            missing.extend(missing_empty.drain(..));
-        }
-
-        SplitConstructorSet { present, missing, missing_empty }
-    }
-}
-
-/// A value can be decomposed into a constructor applied to some fields. This struct represents
-/// those fields, generalized to allow patterns in each field. See also `Constructor`.
-///
-/// This is constructed for a constructor using [`Fields::wildcards()`]. The idea is that
-/// [`Fields::wildcards()`] constructs a list of fields where all entries are wildcards, and then
-/// given a pattern we fill some of the fields with its subpatterns.
-/// In the following example `Fields::wildcards` returns `[_, _, _, _]`. Then in
-/// `extract_pattern_arguments` we fill some of the entries, and the result is
-/// `[Some(0), _, _, _]`.
-/// ```compile_fail,E0004
-/// # fn foo() -> [Option<u8>; 4] { [None; 4] }
-/// let x: [Option<u8>; 4] = foo();
-/// match x {
-///     [Some(0), ..] => {}
-/// }
-/// ```
-///
-/// Note that the number of fields of a constructor may not match the fields declared in the
-/// original struct/variant. This happens if a private or `non_exhaustive` field is uninhabited,
-/// because the code mustn't observe that it is uninhabited. In that case that field is not
-/// included in `fields`. For that reason, when you have a `FieldIdx` you must use
-/// `index_with_declared_idx`.
-#[derive(Debug, Clone, Copy)]
-pub(super) struct Fields<'p, 'tcx> {
-    fields: &'p [DeconstructedPat<'p, 'tcx>],
-}
-
-impl<'p, 'tcx> Fields<'p, 'tcx> {
-    fn empty() -> Self {
-        Fields { fields: &[] }
-    }
-
-    fn singleton(cx: &MatchCheckCtxt<'p, 'tcx>, field: DeconstructedPat<'p, 'tcx>) -> Self {
-        let field: &_ = cx.pattern_arena.alloc(field);
-        Fields { fields: std::slice::from_ref(field) }
-    }
-
-    pub(super) fn from_iter(
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        fields: impl IntoIterator<Item = DeconstructedPat<'p, 'tcx>>,
-    ) -> Self {
-        let fields: &[_] = cx.pattern_arena.alloc_from_iter(fields);
-        Fields { fields }
-    }
-
-    fn wildcards_from_tys(
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        tys: impl IntoIterator<Item = Ty<'tcx>>,
-    ) -> Self {
-        Fields::from_iter(cx, tys.into_iter().map(|ty| DeconstructedPat::wildcard(ty, DUMMY_SP)))
-    }
-
-    // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
-    // uninhabited fields in order not to reveal the uninhabitedness of the whole variant.
-    // This lists the fields we keep along with their types.
-    fn list_variant_nonhidden_fields<'a>(
-        cx: &'a MatchCheckCtxt<'p, 'tcx>,
-        ty: Ty<'tcx>,
-        variant: &'a VariantDef,
-    ) -> impl Iterator<Item = (FieldIdx, Ty<'tcx>)> + Captures<'a> + Captures<'p> {
-        let ty::Adt(adt, args) = ty.kind() else { bug!() };
-        // Whether we must not match the fields of this variant exhaustively.
-        let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
-
-        variant.fields.iter().enumerate().filter_map(move |(i, field)| {
-            let ty = field.ty(cx.tcx, args);
-            // `field.ty()` doesn't normalize after substituting.
-            let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
-            let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
-            let is_uninhabited = cx.tcx.features().exhaustive_patterns && cx.is_uninhabited(ty);
-
-            if is_uninhabited && (!is_visible || is_non_exhaustive) {
-                None
-            } else {
-                Some((FieldIdx::new(i), ty))
-            }
-        })
-    }
-
-    /// Creates a new list of wildcard fields for a given constructor. The result must have a
-    /// length of `constructor.arity()`.
-    #[instrument(level = "trace")]
-    pub(super) fn wildcards(pcx: &PatCtxt<'_, 'p, 'tcx>, constructor: &Constructor<'tcx>) -> Self {
-        let ret = match constructor {
-            Single | Variant(_) => match pcx.ty.kind() {
-                ty::Tuple(fs) => Fields::wildcards_from_tys(pcx.cx, fs.iter()),
-                ty::Ref(_, rty, _) => Fields::wildcards_from_tys(pcx.cx, once(*rty)),
-                ty::Adt(adt, args) => {
-                    if adt.is_box() {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        Fields::wildcards_from_tys(pcx.cx, once(args.type_at(0)))
-                    } else {
-                        let variant = &adt.variant(constructor.variant_index_for_adt(*adt));
-                        let tys = Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant)
-                            .map(|(_, ty)| ty);
-                        Fields::wildcards_from_tys(pcx.cx, tys)
-                    }
-                }
-                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx),
-            },
-            Slice(slice) => match *pcx.ty.kind() {
-                ty::Slice(ty) | ty::Array(ty, _) => {
-                    let arity = slice.arity();
-                    Fields::wildcards_from_tys(pcx.cx, (0..arity).map(|_| ty))
-                }
-                _ => bug!("bad slice pattern {:?} {:?}", constructor, pcx),
-            },
-            Bool(..)
-            | IntRange(..)
-            | F32Range(..)
-            | F64Range(..)
-            | Str(..)
-            | Opaque(..)
-            | NonExhaustive
-            | Hidden
-            | Missing { .. }
-            | Wildcard => Fields::empty(),
-            Or => {
-                bug!("called `Fields::wildcards` on an `Or` ctor")
-            }
-        };
-        debug!(?ret);
-        ret
-    }
-
-    /// Returns the list of patterns.
-    pub(super) fn iter_patterns<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.fields.iter()
-    }
-}
-
-/// Values and patterns can be represented as a constructor applied to some fields. This represents
-/// a pattern in this form.
-/// This also uses interior mutability to keep track of whether the pattern has been found reachable
-/// during analysis. For this reason they cannot be cloned.
-/// A `DeconstructedPat` will almost always come from user input; the only exception are some
-/// `Wildcard`s introduced during specialization.
-pub(crate) struct DeconstructedPat<'p, 'tcx> {
-    ctor: Constructor<'tcx>,
-    fields: Fields<'p, 'tcx>,
-    ty: Ty<'tcx>,
-    span: Span,
-    /// Whether removing this arm would change the behavior of the match expression.
-    useful: Cell<bool>,
-}
-
-impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
-    pub(super) fn wildcard(ty: Ty<'tcx>, span: Span) -> Self {
-        Self::new(Wildcard, Fields::empty(), ty, span)
-    }
-
-    pub(super) fn new(
-        ctor: Constructor<'tcx>,
-        fields: Fields<'p, 'tcx>,
-        ty: Ty<'tcx>,
-        span: Span,
-    ) -> Self {
-        DeconstructedPat { ctor, fields, ty, span, useful: Cell::new(false) }
-    }
-
-    /// Note: the input patterns must have been lowered through
-    /// `super::check_match::MatchVisitor::lower_pattern`.
-    pub(crate) fn from_pat(cx: &MatchCheckCtxt<'p, 'tcx>, pat: &Pat<'tcx>) -> Self {
-        let mkpat = |pat| DeconstructedPat::from_pat(cx, pat);
-        let ctor;
-        let fields;
-        match &pat.kind {
-            PatKind::AscribeUserType { subpattern, .. }
-            | PatKind::InlineConstant { subpattern, .. } => return mkpat(subpattern),
-            PatKind::Binding { subpattern: Some(subpat), .. } => return mkpat(subpat),
-            PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
-                ctor = Wildcard;
-                fields = Fields::empty();
-            }
-            PatKind::Deref { subpattern } => {
-                ctor = Single;
-                fields = Fields::singleton(cx, mkpat(subpattern));
-            }
-            PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => {
-                match pat.ty.kind() {
-                    ty::Tuple(fs) => {
-                        ctor = Single;
-                        let mut wilds: SmallVec<[_; 2]> =
-                            fs.iter().map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
-                        for pat in subpatterns {
-                            wilds[pat.field.index()] = mkpat(&pat.pattern);
-                        }
-                        fields = Fields::from_iter(cx, wilds);
-                    }
-                    ty::Adt(adt, args) if adt.is_box() => {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
-                        // _)` or a box pattern. As a hack to avoid an ICE with the former, we
-                        // ignore other fields than the first one. This will trigger an error later
-                        // anyway.
-                        // See https://github.com/rust-lang/rust/issues/82772 ,
-                        // explanation: https://github.com/rust-lang/rust/pull/82789#issuecomment-796921977
-                        // The problem is that we can't know from the type whether we'll match
-                        // normally or through box-patterns. We'll have to figure out a proper
-                        // solution when we introduce generalized deref patterns. Also need to
-                        // prevent mixing of those two options.
-                        let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
-                        let pat = if let Some(pat) = pattern {
-                            mkpat(&pat.pattern)
-                        } else {
-                            DeconstructedPat::wildcard(args.type_at(0), pat.span)
-                        };
-                        ctor = Single;
-                        fields = Fields::singleton(cx, pat);
-                    }
-                    ty::Adt(adt, _) => {
-                        ctor = match pat.kind {
-                            PatKind::Leaf { .. } => Single,
-                            PatKind::Variant { variant_index, .. } => Variant(variant_index),
-                            _ => bug!(),
-                        };
-                        let variant = &adt.variant(ctor.variant_index_for_adt(*adt));
-                        // For each field in the variant, we store the relevant index into `self.fields` if any.
-                        let mut field_id_to_id: Vec<Option<usize>> =
-                            (0..variant.fields.len()).map(|_| None).collect();
-                        let tys = Fields::list_variant_nonhidden_fields(cx, pat.ty, variant)
-                            .enumerate()
-                            .map(|(i, (field, ty))| {
-                                field_id_to_id[field.index()] = Some(i);
-                                ty
-                            });
-                        let mut wilds: SmallVec<[_; 2]> =
-                            tys.map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
-                        for pat in subpatterns {
-                            if let Some(i) = field_id_to_id[pat.field.index()] {
-                                wilds[i] = mkpat(&pat.pattern);
-                            }
-                        }
-                        fields = Fields::from_iter(cx, wilds);
-                    }
-                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
-                }
-            }
-            PatKind::Constant { value } => {
-                match pat.ty.kind() {
-                    ty::Bool => {
-                        ctor = match value.try_eval_bool(cx.tcx, cx.param_env) {
-                            Some(b) => Bool(b),
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Char | ty::Int(_) | ty::Uint(_) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => IntRange(IntRange::from_bits(cx.tcx, pat.ty, bits)),
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Float(ty::FloatTy::F32) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => {
-                                use rustc_apfloat::Float;
-                                let value = rustc_apfloat::ieee::Single::from_bits(bits);
-                                F32Range(value, value, RangeEnd::Included)
-                            }
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Float(ty::FloatTy::F64) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => {
-                                use rustc_apfloat::Float;
-                                let value = rustc_apfloat::ieee::Double::from_bits(bits);
-                                F64Range(value, value, RangeEnd::Included)
-                            }
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Ref(_, t, _) if t.is_str() => {
-                        // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
-                        // with other `Deref` patterns. This could have been done in `const_to_pat`,
-                        // but that causes issues with the rest of the matching code.
-                        // So here, the constructor for a `"foo"` pattern is `&` (represented by
-                        // `Single`), and has one field. That field has constructor `Str(value)` and no
-                        // fields.
-                        // Note: `t` is `str`, not `&str`.
-                        let subpattern =
-                            DeconstructedPat::new(Str(*value), Fields::empty(), *t, pat.span);
-                        ctor = Single;
-                        fields = Fields::singleton(cx, subpattern)
-                    }
-                    // All constants that can be structurally matched have already been expanded
-                    // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
-                    // opaque.
-                    _ => {
-                        ctor = Opaque(OpaqueId::new());
-                        fields = Fields::empty();
-                    }
-                }
-            }
-            PatKind::Range(box PatRange { lo, hi, end, .. }) => {
-                let ty = pat.ty;
-                ctor = match ty.kind() {
-                    ty::Char | ty::Int(_) | ty::Uint(_) => {
-                        let lo =
-                            MaybeInfiniteInt::from_pat_range_bdy(*lo, ty, cx.tcx, cx.param_env);
-                        let hi =
-                            MaybeInfiniteInt::from_pat_range_bdy(*hi, ty, cx.tcx, cx.param_env);
-                        IntRange(IntRange::from_range(lo, hi, *end))
-                    }
-                    ty::Float(fty) => {
-                        use rustc_apfloat::Float;
-                        let lo = lo.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
-                        let hi = hi.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
-                        match fty {
-                            ty::FloatTy::F32 => {
-                                use rustc_apfloat::ieee::Single;
-                                let lo = lo.map(Single::from_bits).unwrap_or(-Single::INFINITY);
-                                let hi = hi.map(Single::from_bits).unwrap_or(Single::INFINITY);
-                                F32Range(lo, hi, *end)
-                            }
-                            ty::FloatTy::F64 => {
-                                use rustc_apfloat::ieee::Double;
-                                let lo = lo.map(Double::from_bits).unwrap_or(-Double::INFINITY);
-                                let hi = hi.map(Double::from_bits).unwrap_or(Double::INFINITY);
-                                F64Range(lo, hi, *end)
-                            }
-                        }
-                    }
-                    _ => bug!("invalid type for range pattern: {}", ty),
-                };
-                fields = Fields::empty();
-            }
-            PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
-                let array_len = match pat.ty.kind() {
-                    ty::Array(_, length) => {
-                        Some(length.eval_target_usize(cx.tcx, cx.param_env) as usize)
-                    }
-                    ty::Slice(_) => None,
-                    _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
-                };
-                let kind = if slice.is_some() {
-                    VarLen(prefix.len(), suffix.len())
-                } else {
-                    FixedLen(prefix.len() + suffix.len())
-                };
-                ctor = Slice(Slice::new(array_len, kind));
-                fields =
-                    Fields::from_iter(cx, prefix.iter().chain(suffix.iter()).map(|p| mkpat(&*p)));
-            }
-            PatKind::Or { .. } => {
-                ctor = Or;
-                let pats = expand_or_pat(pat);
-                fields = Fields::from_iter(cx, pats.into_iter().map(mkpat));
-            }
-            PatKind::Never => {
-                // FIXME(never_patterns): handle `!` in exhaustiveness. This is a sane default
-                // in the meantime.
-                ctor = Wildcard;
-                fields = Fields::empty();
-            }
-            PatKind::Error(_) => {
-                ctor = Opaque(OpaqueId::new());
-                fields = Fields::empty();
-            }
-        }
-        DeconstructedPat::new(ctor, fields, pat.ty, pat.span)
-    }
-
-    pub(super) fn is_or_pat(&self) -> bool {
-        matches!(self.ctor, Or)
-    }
-    /// Expand this (possibly-nested) or-pattern into its alternatives.
-    pub(super) fn flatten_or_pat(&'p self) -> SmallVec<[&'p Self; 1]> {
-        if self.is_or_pat() {
-            self.iter_fields().flat_map(|p| p.flatten_or_pat()).collect()
-        } else {
-            smallvec![self]
-        }
-    }
-
-    pub(super) fn ctor(&self) -> &Constructor<'tcx> {
-        &self.ctor
-    }
-    pub(super) fn ty(&self) -> Ty<'tcx> {
-        self.ty
-    }
-    pub(super) fn span(&self) -> Span {
-        self.span
-    }
-
-    pub(super) fn iter_fields<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.fields.iter_patterns()
-    }
-
-    /// Specialize this pattern with a constructor.
-    /// `other_ctor` can be different from `self.ctor`, but must be covered by it.
-    pub(super) fn specialize<'a>(
-        &'a self,
-        pcx: &PatCtxt<'_, 'p, 'tcx>,
-        other_ctor: &Constructor<'tcx>,
-    ) -> SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]> {
-        match (&self.ctor, other_ctor) {
-            (Wildcard, _) => {
-                // We return a wildcard for each field of `other_ctor`.
-                Fields::wildcards(pcx, other_ctor).iter_patterns().collect()
-            }
-            (Slice(self_slice), Slice(other_slice))
-                if self_slice.arity() != other_slice.arity() =>
-            {
-                // The only tricky case: two slices of different arity. Since `self_slice` covers
-                // `other_slice`, `self_slice` must be `VarLen`, i.e. of the form
-                // `[prefix, .., suffix]`. Moreover `other_slice` is guaranteed to have a larger
-                // arity. So we fill the middle part with enough wildcards to reach the length of
-                // the new, larger slice.
-                match self_slice.kind {
-                    FixedLen(_) => bug!("{:?} doesn't cover {:?}", self_slice, other_slice),
-                    VarLen(prefix, suffix) => {
-                        let (ty::Slice(inner_ty) | ty::Array(inner_ty, _)) = *self.ty.kind() else {
-                            bug!("bad slice pattern {:?} {:?}", self.ctor, self.ty);
-                        };
-                        let prefix = &self.fields.fields[..prefix];
-                        let suffix = &self.fields.fields[self_slice.arity() - suffix..];
-                        let wildcard: &_ = pcx
-                            .cx
-                            .pattern_arena
-                            .alloc(DeconstructedPat::wildcard(inner_ty, DUMMY_SP));
-                        let extra_wildcards = other_slice.arity() - self_slice.arity();
-                        let extra_wildcards = (0..extra_wildcards).map(|_| wildcard);
-                        prefix.iter().chain(extra_wildcards).chain(suffix).collect()
-                    }
-                }
-            }
-            _ => self.fields.iter_patterns().collect(),
-        }
-    }
-
-    /// We keep track for each pattern if it was ever useful during the analysis. This is used
-    /// with `redundant_spans` to report redundant subpatterns arising from or patterns.
-    pub(super) fn set_useful(&self) {
-        self.useful.set(true)
-    }
-    pub(super) fn is_useful(&self) -> bool {
-        if self.useful.get() {
-            true
-        } else if self.is_or_pat() && self.iter_fields().any(|f| f.is_useful()) {
-            // We always expand or patterns in the matrix, so we will never see the actual
-            // or-pattern (the one with constructor `Or`) in the column. As such, it will not be
-            // marked as useful itself, only its children will. We recover this information here.
-            self.set_useful();
-            true
-        } else {
-            false
-        }
-    }
-
-    /// Report the spans of subpatterns that were not useful, if any.
-    pub(super) fn redundant_spans(&self) -> Vec<Span> {
-        let mut spans = Vec::new();
-        self.collect_redundant_spans(&mut spans);
-        spans
-    }
-    fn collect_redundant_spans(&self, spans: &mut Vec<Span>) {
-        // We don't look at subpatterns if we already reported the whole pattern as redundant.
-        if !self.is_useful() {
-            spans.push(self.span);
-        } else {
-            for p in self.iter_fields() {
-                p.collect_redundant_spans(spans);
-            }
-        }
-    }
-}
-
-/// This is mostly copied from the `Pat` impl. This is best effort and not good enough for a
-/// `Display` impl.
-impl<'p, 'tcx> fmt::Debug for DeconstructedPat<'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // Printing lists is a chore.
-        let mut first = true;
-        let mut start_or_continue = |s| {
-            if first {
-                first = false;
-                ""
-            } else {
-                s
-            }
-        };
-        let mut start_or_comma = || start_or_continue(", ");
-
-        match &self.ctor {
-            Single | Variant(_) => match self.ty.kind() {
-                ty::Adt(def, _) if def.is_box() => {
-                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
-                    // of `std`). So this branch is only reachable when the feature is enabled and
-                    // the pattern is a box pattern.
-                    let subpattern = self.iter_fields().next().unwrap();
-                    write!(f, "box {subpattern:?}")
-                }
-                ty::Adt(..) | ty::Tuple(..) => {
-                    let variant = match self.ty.kind() {
-                        ty::Adt(adt, _) => Some(adt.variant(self.ctor.variant_index_for_adt(*adt))),
-                        ty::Tuple(_) => None,
-                        _ => unreachable!(),
-                    };
-
-                    if let Some(variant) = variant {
-                        write!(f, "{}", variant.name)?;
-                    }
-
-                    // Without `cx`, we can't know which field corresponds to which, so we can't
-                    // get the names of the fields. Instead we just display everything as a tuple
-                    // struct, which should be good enough.
-                    write!(f, "(")?;
-                    for p in self.iter_fields() {
-                        write!(f, "{}", start_or_comma())?;
-                        write!(f, "{p:?}")?;
-                    }
-                    write!(f, ")")
-                }
-                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
-                // be careful to detect strings here. However a string literal pattern will never
-                // be reported as a non-exhaustiveness witness, so we can ignore this issue.
-                ty::Ref(_, _, mutbl) => {
-                    let subpattern = self.iter_fields().next().unwrap();
-                    write!(f, "&{}{:?}", mutbl.prefix_str(), subpattern)
-                }
-                _ => write!(f, "_"),
-            },
-            Slice(slice) => {
-                let mut subpatterns = self.fields.iter_patterns();
-                write!(f, "[")?;
-                match slice.kind {
-                    FixedLen(_) => {
-                        for p in subpatterns {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                    }
-                    VarLen(prefix_len, _) => {
-                        for p in subpatterns.by_ref().take(prefix_len) {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                        write!(f, "{}", start_or_comma())?;
-                        write!(f, "..")?;
-                        for p in subpatterns {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                    }
-                }
-                write!(f, "]")
-            }
-            Bool(b) => write!(f, "{b}"),
-            // Best-effort, will render signed ranges incorrectly
-            IntRange(range) => write!(f, "{range:?}"),
-            F32Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
-            F64Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
-            Str(value) => write!(f, "{value}"),
-            Opaque(..) => write!(f, "<constant pattern>"),
-            Or => {
-                for pat in self.iter_fields() {
-                    write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
-                }
-                Ok(())
-            }
-            Wildcard | Missing { .. } | NonExhaustive | Hidden => write!(f, "_ : {:?}", self.ty),
-        }
-    }
-}
-
-/// Same idea as `DeconstructedPat`, except this is a fictitious pattern built up for diagnostics
-/// purposes. As such they don't use interning and can be cloned.
-#[derive(Debug, Clone)]
-pub(crate) struct WitnessPat<'tcx> {
-    ctor: Constructor<'tcx>,
-    pub(crate) fields: Vec<WitnessPat<'tcx>>,
-    ty: Ty<'tcx>,
-}
-
-impl<'tcx> WitnessPat<'tcx> {
-    pub(super) fn new(ctor: Constructor<'tcx>, fields: Vec<Self>, ty: Ty<'tcx>) -> Self {
-        Self { ctor, fields, ty }
-    }
-    pub(super) fn wildcard(ty: Ty<'tcx>) -> Self {
-        Self::new(Wildcard, Vec::new(), ty)
-    }
-
-    /// Construct a pattern that matches everything that starts with this constructor.
-    /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
-    /// `Some(_)`.
-    pub(super) fn wild_from_ctor(pcx: &PatCtxt<'_, '_, 'tcx>, ctor: Constructor<'tcx>) -> Self {
-        // Reuse `Fields::wildcards` to get the types.
-        let fields = Fields::wildcards(pcx, &ctor)
-            .iter_patterns()
-            .map(|deco_pat| Self::wildcard(deco_pat.ty()))
-            .collect();
-        Self::new(ctor, fields, pcx.ty)
-    }
-
-    pub(super) fn ctor(&self) -> &Constructor<'tcx> {
-        &self.ctor
-    }
-    pub(super) fn ty(&self) -> Ty<'tcx> {
-        self.ty
-    }
-
-    /// Convert back to a `thir::Pat` for diagnostic purposes. This panics for patterns that don't
-    /// appear in diagnostics, like float ranges.
-    pub(crate) fn to_diagnostic_pat(&self, cx: &MatchCheckCtxt<'_, 'tcx>) -> Pat<'tcx> {
-        let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
-        let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_diagnostic_pat(cx)));
-        let kind = match &self.ctor {
-            Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
-            IntRange(range) => return range.to_diagnostic_pat(self.ty, cx.tcx),
-            Single | Variant(_) => match self.ty.kind() {
-                ty::Tuple(..) => PatKind::Leaf {
-                    subpatterns: subpatterns
-                        .enumerate()
-                        .map(|(i, pattern)| FieldPat { field: FieldIdx::new(i), pattern })
-                        .collect(),
-                },
-                ty::Adt(adt_def, _) if adt_def.is_box() => {
-                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
-                    // of `std`). So this branch is only reachable when the feature is enabled and
-                    // the pattern is a box pattern.
-                    PatKind::Deref { subpattern: subpatterns.next().unwrap() }
-                }
-                ty::Adt(adt_def, args) => {
-                    let variant_index = self.ctor.variant_index_for_adt(*adt_def);
-                    let variant = &adt_def.variant(variant_index);
-                    let subpatterns = Fields::list_variant_nonhidden_fields(cx, self.ty, variant)
-                        .zip(subpatterns)
-                        .map(|((field, _ty), pattern)| FieldPat { field, pattern })
-                        .collect();
-
-                    if adt_def.is_enum() {
-                        PatKind::Variant { adt_def: *adt_def, args, variant_index, subpatterns }
-                    } else {
-                        PatKind::Leaf { subpatterns }
-                    }
-                }
-                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
-                // be careful to reconstruct the correct constant pattern here. However a string
-                // literal pattern will never be reported as a non-exhaustiveness witness, so we
-                // ignore this issue.
-                ty::Ref(..) => PatKind::Deref { subpattern: subpatterns.next().unwrap() },
-                _ => bug!("unexpected ctor for type {:?} {:?}", self.ctor, self.ty),
-            },
-            Slice(slice) => {
-                match slice.kind {
-                    FixedLen(_) => PatKind::Slice {
-                        prefix: subpatterns.collect(),
-                        slice: None,
-                        suffix: Box::new([]),
-                    },
-                    VarLen(prefix, _) => {
-                        let mut subpatterns = subpatterns.peekable();
-                        let mut prefix: Vec<_> = subpatterns.by_ref().take(prefix).collect();
-                        if slice.array_len.is_some() {
-                            // Improves diagnostics a bit: if the type is a known-size array, instead
-                            // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
-                            // This is incorrect if the size is not known, since `[_, ..]` captures
-                            // arrays of lengths `>= 1` whereas `[..]` captures any length.
-                            while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
-                                prefix.pop();
-                            }
-                            while subpatterns.peek().is_some()
-                                && is_wildcard(subpatterns.peek().unwrap())
-                            {
-                                subpatterns.next();
-                            }
-                        }
-                        let suffix: Box<[_]> = subpatterns.collect();
-                        let wild = Pat::wildcard_from_ty(self.ty);
-                        PatKind::Slice {
-                            prefix: prefix.into_boxed_slice(),
-                            slice: Some(Box::new(wild)),
-                            suffix,
-                        }
-                    }
-                }
-            }
-            &Str(value) => PatKind::Constant { value },
-            Wildcard | NonExhaustive | Hidden => PatKind::Wild,
-            Missing { .. } => bug!(
-                "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
-                `Missing` should have been processed in `apply_constructors`"
-            ),
-            F32Range(..) | F64Range(..) | Opaque(..) | Or => {
-                bug!("can't convert to pattern: {:?}", self)
-            }
-        };
-
-        Pat { ty: self.ty, span: DUMMY_SP, kind }
-    }
-
-    pub(super) fn iter_fields<'a>(&'a self) -> impl Iterator<Item = &'a WitnessPat<'tcx>> {
-        self.fields.iter()
-    }
-}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index eb548ad29eb..af0dab4ebc7 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -2,11 +2,8 @@
 
 mod check_match;
 mod const_to_pat;
-pub(crate) mod deconstruct_pat;
-mod usefulness;
 
 pub(crate) use self::check_match::check_match;
-pub(crate) use self::usefulness::MatchCheckCtxt;
 
 use crate::errors::*;
 use crate::thir::util::UserAnnotatedTyHelpers;
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
deleted file mode 100644
index 637cc38be2c..00000000000
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ /dev/null
@@ -1,1648 +0,0 @@
-//! # Match exhaustiveness and redundancy algorithm
-//!
-//! This file contains the logic for exhaustiveness and usefulness checking for pattern-matching.
-//! Specifically, given a list of patterns in a match, we can tell whether:
-//! (a) a given pattern is redundant
-//! (b) the patterns cover every possible value for the type (exhaustiveness)
-//!
-//! The algorithm implemented here is inspired from the one described in [this
-//! paper](http://moscova.inria.fr/~maranget/papers/warn/index.html). We have however changed it in
-//! various ways to accommodate the variety of patterns that Rust supports. We thus explain our
-//! version here, without being as precise.
-//!
-//! Fun fact: computing exhaustiveness is NP-complete, because we can encode a SAT problem as an
-//! exhaustiveness problem. See [here](https://niedzejkob.p4.team/rust-np) for the fun details.
-//!
-//!
-//! # Summary
-//!
-//! The algorithm is given as input a list of patterns, one for each arm of a match, and computes
-//! the following:
-//! - a set of values that match none of the patterns (if any),
-//! - for each subpattern (taking into account or-patterns), whether removing it would change
-//!     anything about how the match executes, i.e. whether it is useful/not redundant.
-//!
-//! To a first approximation, the algorithm works by exploring all possible values for the type
-//! being matched on, and determining which arm(s) catch which value. To make this tractable we
-//! cleverly group together values, as we'll see below.
-//!
-//! The entrypoint of this file is the [`compute_match_usefulness`] function, which computes
-//! usefulness for each subpattern and exhaustiveness for the whole match.
-//!
-//! In this page we explain the necessary concepts to understand how the algorithm works.
-//!
-//!
-//! # Usefulness
-//!
-//! The central concept of this file is the notion of "usefulness". Given some patterns `p_1 ..
-//! p_n`, a pattern `q` is said to be *useful* if there is a value that is matched by `q` and by
-//! none of the `p_i`. We write `usefulness(p_1 .. p_n, q)` for a function that returns a list of
-//! such values. The aim of this file is to compute it efficiently.
-//!
-//! This is enough to compute usefulness: a pattern in a `match` expression is redundant iff it is
-//! not useful w.r.t. the patterns above it:
-//! ```compile_fail,E0004
-//! # #![feature(exclusive_range_pattern)]
-//! # fn foo() {
-//! match Some(0u32) {
-//!     Some(0..100) => {},
-//!     Some(90..190) => {}, // useful: `Some(150)` is matched by this but not the branch above
-//!     Some(50..150) => {}, // redundant: all the values this matches are already matched by
-//!                          //   the branches above
-//!     None => {},          // useful: `None` is matched by this but not the branches above
-//! }
-//! # }
-//! ```
-//!
-//! This is also enough to compute exhaustiveness: a match is exhaustive iff the wildcard `_`
-//! pattern is _not_ useful w.r.t. the patterns in the match. The values returned by `usefulness`
-//! are used to tell the user which values are missing.
-//! ```compile_fail,E0004
-//! # fn foo(x: Option<u32>) {
-//! match x {
-//!     None => {},
-//!     Some(0) => {},
-//!     // not exhaustive: `_` is useful because it matches `Some(1)`
-//! }
-//! # }
-//! ```
-//!
-//!
-//! # Constructors and fields
-//!
-//! In the value `Pair(Some(0), true)`, `Pair` is called the constructor of the value, and `Some(0)`
-//! and `true` are its fields. Every matcheable value can be decomposed in this way. Examples of
-//! constructors are: `Some`, `None`, `(,)` (the 2-tuple constructor), `Foo {..}` (the constructor
-//! for a struct `Foo`), and `2` (the constructor for the number `2`).
-//!
-//! Each constructor takes a fixed number of fields; this is called its arity. `Pair` and `(,)` have
-//! arity 2, `Some` has arity 1, `None` and `42` have arity 0. Each type has a known set of
-//! constructors. Some types have many constructors (like `u64`) or even an infinitely many (like
-//! `&str` and `&[T]`).
-//!
-//! Patterns are similar: `Pair(Some(_), _)` has constructor `Pair` and two fields. The difference
-//! is that we get some extra pattern-only constructors, namely: the wildcard `_`, variable
-//! bindings, integer ranges like `0..=10`, and variable-length slices like `[_, .., _]`. We treat
-//! or-patterns separately, see the dedicated section below.
-//!
-//! Now to check if a value `v` matches a pattern `p`, we check if `v`'s constructor matches `p`'s
-//! constructor, then recursively compare their fields if necessary. A few representative examples:
-//!
-//! - `matches!(v, _) := true`
-//! - `matches!((v0,  v1), (p0,  p1)) := matches!(v0, p0) && matches!(v1, p1)`
-//! - `matches!(Foo { bar: v0, baz: v1 }, Foo { bar: p0, baz: p1 }) := matches!(v0, p0) && matches!(v1, p1)`
-//! - `matches!(Ok(v0), Ok(p0)) := matches!(v0, p0)`
-//! - `matches!(Ok(v0), Err(p0)) := false` (incompatible variants)
-//! - `matches!(v, 1..=100) := matches!(v, 1) || ... || matches!(v, 100)`
-//! - `matches!([v0], [p0, .., p1]) := false` (incompatible lengths)
-//! - `matches!([v0, v1, v2], [p0, .., p1]) := matches!(v0, p0) && matches!(v2, p1)`
-//!
-//! Constructors, fields and relevant operations are defined in the [`super::deconstruct_pat`]
-//! module. The question of whether a constructor is matched by another one is answered by
-//! [`Constructor::is_covered_by`].
-//!
-//! Note 1: variable bindings (like the `x` in `Some(x)`) match anything, so we treat them as wildcards.
-//! Note 2: this only applies to matcheable values. For example a value of type `Rc<u64>` can't be
-//! deconstructed that way.
-//!
-//!
-//!
-//! # Specialization
-//!
-//! The examples in the previous section motivate the operation at the heart of the algorithm:
-//! "specialization". It captures this idea of "removing one layer of constructor".
-//!
-//! `specialize(c, p)` takes a value-only constructor `c` and a pattern `p`, and returns a
-//! pattern-tuple or nothing. It works as follows:
-//!
-//! - Specializing for the wrong constructor returns nothing
-//!
-//!   - `specialize(None, Some(p0)) := <nothing>`
-//!   - `specialize([,,,], [p0]) := <nothing>`
-//!
-//! - Specializing for the correct constructor returns a tuple of the fields
-//!
-//!   - `specialize(Variant1, Variant1(p0, p1, p2)) := (p0, p1, p2)`
-//!   - `specialize(Foo{ bar, baz, quz }, Foo { bar: p0, baz: p1, .. }) := (p0, p1, _)`
-//!   - `specialize([,,,], [p0, .., p1]) := (p0, _, _, p1)`
-//!
-//! We get the following property: for any values `v_1, .., v_n` of appropriate types, we have:
-//! ```text
-//! matches!(c(v_1, .., v_n), p)
-//! <=> specialize(c, p) returns something
-//!     && matches!((v_1, .., v_n), specialize(c, p))
-//! ```
-//!
-//! We also extend specialization to pattern-tuples by applying it to the first pattern:
-//! `specialize(c, (p_0, .., p_n)) := specialize(c, p_0) ++ (p_1, .., p_m)`
-//! where `++` is concatenation of tuples.
-//!
-//!
-//! The previous property extends to pattern-tuples:
-//! ```text
-//! matches!((c(v_1, .., v_n), w_1, .., w_m), (p_0, p_1, .., p_m))
-//! <=> specialize(c, p_0) does not error
-//!     && matches!((v_1, .., v_n, w_1, .., w_m), specialize(c, (p_0, p_1, .., p_m)))
-//! ```
-//!
-//! Whether specialization returns something or not is given by [`Constructor::is_covered_by`].
-//! Specialization of a pattern is computed in [`DeconstructedPat::specialize`]. Specialization for
-//! a pattern-tuple is computed in [`PatStack::pop_head_constructor`]. Finally, specialization for a
-//! set of pattern-tuples is computed in [`Matrix::specialize_constructor`].
-//!
-//!
-//!
-//! # Undoing specialization
-//!
-//! To construct witnesses we will need an inverse of specialization. If `c` is a constructor of
-//! arity `n`, we define `unspecialize` as:
-//! `unspecialize(c, (p_1, .., p_n, q_1, .., q_m)) := (c(p_1, .., p_n), q_1, .., q_m)`.
-//!
-//! This is done for a single witness-tuple in [`WitnessStack::apply_constructor`], and for a set of
-//! witness-tuples in [`WitnessMatrix::apply_constructor`].
-//!
-//!
-//!
-//! # Computing usefulness
-//!
-//! We now present a naive version of the algorithm for computing usefulness. From now on we operate
-//! on pattern-tuples.
-//!
-//! Let `pt_1, .., pt_n` and `qt` be length-m tuples of patterns for the same type `(T_1, .., T_m)`.
-//! We compute `usefulness(tp_1, .., tp_n, tq)` as follows:
-//!
-//! - Base case: `m == 0`.
-//!     The pattern-tuples are all empty, i.e. they're all `()`. Thus `tq` is useful iff there are
-//!     no rows above it, i.e. if `n == 0`. In that case we return `()` as a witness-tuple of
-//!     usefulness of `tq`.
-//!
-//! - Inductive case: `m > 0`.
-//!     In this naive version, we list all the possible constructors for values of type `T1` (we
-//!     will be more clever in the next section).
-//!
-//!     - For each such constructor `c` for which `specialize(c, tq)` is not nothing:
-//!         - We recursively compute `usefulness(specialize(c, tp_1) ... specialize(c, tp_n), specialize(c, tq))`,
-//!             where we discard any `specialize(c, p_i)` that returns nothing.
-//!         - For each witness-tuple `w` found, we apply `unspecialize(c, w)` to it.
-//!
-//!     - We return the all the witnesses found, if any.
-//!
-//!
-//! Let's take the following example:
-//! ```compile_fail,E0004
-//! # enum Enum { Variant1(()), Variant2(Option<bool>, u32)}
-//! # use Enum::*;
-//! # fn foo(x: Enum) {
-//! match x {
-//!     Variant1(_) => {} // `p1`
-//!     Variant2(None, 0) => {} // `p2`
-//!     Variant2(Some(_), 0) => {} // `q`
-//! }
-//! # }
-//! ```
-//!
-//! To compute the usefulness of `q`, we would proceed as follows:
-//! ```text
-//! Start:
-//!   `tp1 = [Variant1(_)]`
-//!   `tp2 = [Variant2(None, 0)]`
-//!   `tq  = [Variant2(Some(true), 0)]`
-//!
-//!   Constructors are `Variant1` and `Variant2`. Only `Variant2` can specialize `tq`.
-//!   Specialize with `Variant2`:
-//!     `tp2 = [None, 0]`
-//!     `tq  = [Some(true), 0]`
-//!
-//!     Constructors are `None` and `Some`. Only `Some` can specialize `tq`.
-//!     Specialize with `Some`:
-//!       `tq  = [true, 0]`
-//!
-//!       Constructors are `false` and `true`. Only `true` can specialize `tq`.
-//!       Specialize with `true`:
-//!         `tq  = [0]`
-//!
-//!         Constructors are `0`, `1`, .. up to infinity. Only `0` can specialize `tq`.
-//!         Specialize with `0`:
-//!           `tq  = []`
-//!
-//!           m == 0 and n == 0, so `tq` is useful with witness `[]`.
-//!             `witness  = []`
-//!
-//!         Unspecialize with `0`:
-//!           `witness  = [0]`
-//!       Unspecialize with `true`:
-//!         `witness  = [true, 0]`
-//!     Unspecialize with `Some`:
-//!       `witness  = [Some(true), 0]`
-//!   Unspecialize with `Variant2`:
-//!     `witness  = [Variant2(Some(true), 0)]`
-//! ```
-//!
-//! Therefore `usefulness(tp_1, tp_2, tq)` returns the single witness-tuple `[Variant2(Some(true), 0)]`.
-//!
-//!
-//! Computing the set of constructors for a type is done in [`ConstructorSet::for_ty`]. See the
-//! following sections for more accurate versions of the algorithm and corresponding links.
-//!
-//!
-//!
-//! # Computing usefulness and exhaustiveness in one go
-//!
-//! The algorithm we have described so far computes usefulness of each pattern in turn, and ends by
-//! checking if `_` is useful to determine exhaustiveness of the whole match. In practice, instead
-//! of doing "for each pattern { for each constructor { ... } }", we do "for each constructor { for
-//! each pattern { ... } }". This allows us to compute everything in one go.
-//!
-//! [`Matrix`] stores the set of pattern-tuples under consideration. We track usefulness of each
-//! row mutably in the matrix as we go along. We ignore witnesses of usefulness of the match rows.
-//! We gather witnesses of the usefulness of `_` in [`WitnessMatrix`]. The algorithm that computes
-//! all this is in [`compute_exhaustiveness_and_usefulness`].
-//!
-//! See the full example at the bottom of this documentation.
-//!
-//!
-//!
-//! # Making usefulness tractable: constructor splitting
-//!
-//! We're missing one last detail: which constructors do we list? Naively listing all value
-//! constructors cannot work for types like `u64` or `&str`, so we need to be more clever. The final
-//! clever idea for this algorithm is that we can group together constructors that behave the same.
-//!
-//! Examples:
-//! ```compile_fail,E0004
-//! match (0, false) {
-//!     (0 ..=100, true) => {}
-//!     (50..=150, false) => {}
-//!     (0 ..=200, _) => {}
-//! }
-//! ```
-//!
-//! In this example, trying any of `0`, `1`, .., `49` will give the same specialized matrix, and
-//! thus the same usefulness/exhaustiveness results. We can thus accelerate the algorithm by
-//! trying them all at once. Here in fact, the only cases we need to consider are: `0..50`,
-//! `50..=100`, `101..=150`,`151..=200` and `201..`.
-//!
-//! ```
-//! enum Direction { North, South, East, West }
-//! # let wind = (Direction::North, 0u8);
-//! match wind {
-//!     (Direction::North, 50..) => {}
-//!     (_, _) => {}
-//! }
-//! ```
-//!
-//! In this example, trying any of `South`, `East`, `West` will give the same specialized matrix. By
-//! the same reasoning, we only need to try two cases: `North`, and "everything else".
-//!
-//! We call _constructor splitting_ the operation that computes such a minimal set of cases to try.
-//! This is done in [`ConstructorSet::split`] and explained in [`super::deconstruct_pat`].
-//!
-//!
-//!
-//! # Or-patterns
-//!
-//! What we have described so far works well if there are no or-patterns. To handle them, if the
-//! first pattern of a row in the matrix is an or-pattern, we expand it by duplicating the rest of
-//! the row as necessary. This is handled automatically in [`Matrix`].
-//!
-//! This makes usefulness tracking subtle, because we also want to compute whether an alternative
-//! of an or-pattern is redundant, e.g. in `Some(_) | Some(0)`. We track usefulness of each
-//! subpattern by interior mutability in [`DeconstructedPat`] with `set_useful`/`is_useful`.
-//!
-//! It's unfortunate that we have to use interior mutability, but believe me (Nadrieril), I have
-//! tried [other](https://github.com/rust-lang/rust/pull/80104)
-//! [solutions](https://github.com/rust-lang/rust/pull/80632) and nothing is remotely as simple.
-//!
-//!
-//!
-//! # Constants and opaques
-//!
-//! There are two kinds of constants in patterns:
-//!
-//! * literals (`1`, `true`, `"foo"`)
-//! * named or inline consts (`FOO`, `const { 5 + 6 }`)
-//!
-//! The latter are converted into the corresponding patterns by a previous phase. For example
-//! `const_to_pat(const { [1, 2, 3] })` becomes an `Array(vec![Const(1), Const(2), Const(3)])`
-//! pattern. This gets problematic when comparing the constant via `==` would behave differently
-//! from matching on the constant converted to a pattern. The situation around this is currently
-//! unclear and the lang team is working on clarifying what we want to do there. In any case, there
-//! are constants we will not turn into patterns. We capture these with `Constructor::Opaque`. These
-//! `Opaque` patterns do not participate in exhaustiveness, specialization or overlap checking.
-//!
-//!
-//!
-//! # Usefulness vs reachability, validity, and empty patterns
-//!
-//! This is likely the subtlest aspect of the algorithm. To be fully precise, a match doesn't
-//! operate on a value, it operates on a place. In certain unsafe circumstances, it is possible for
-//! a place to not contain valid data for its type. This has subtle consequences for empty types.
-//! Take the following:
-//!
-//! ```rust
-//! enum Void {}
-//! let x: u8 = 0;
-//! let ptr: *const Void = &x as *const u8 as *const Void;
-//! unsafe {
-//!     match *ptr {
-//!         _ => println!("Reachable!"),
-//!     }
-//! }
-//! ```
-//!
-//! In this example, `ptr` is a valid pointer pointing to a place with invalid data. The `_` pattern
-//! does not look at the contents of `*ptr`, so this is ok and the arm is taken. In other words,
-//! despite the place we are inspecting being of type `Void`, there is a reachable arm. If the
-//! arm had a binding however:
-//!
-//! ```rust
-//! # #[derive(Copy, Clone)]
-//! # enum Void {}
-//! # let x: u8 = 0;
-//! # let ptr: *const Void = &x as *const u8 as *const Void;
-//! # unsafe {
-//! match *ptr {
-//!     _a => println!("Unreachable!"),
-//! }
-//! # }
-//! ```
-//!
-//! Here the binding loads the value of type `Void` from the `*ptr` place. In this example, this
-//! causes UB since the data is not valid. In the general case, this asserts validity of the data at
-//! `*ptr`. Either way, this arm will never be taken.
-//!
-//! Finally, let's consider the empty match `match *ptr {}`. If we consider this exhaustive, then
-//! having invalid data at `*ptr` is invalid. In other words, the empty match is semantically
-//! equivalent to the `_a => ...` match. In the interest of explicitness, we prefer the case with an
-//! arm, hence we won't tell the user to remove the `_a` arm. In other words, the `_a` arm is
-//! unreachable yet not redundant. This is why we lint on redundant arms rather than unreachable
-//! arms, despite the fact that the lint says "unreachable".
-//!
-//! These considerations only affects certain places, namely those that can contain non-valid data
-//! without UB. These are: pointer dereferences, reference dereferences, and union field accesses.
-//! We track in the algorithm whether a given place is known to contain valid data. This is done
-//! first by inspecting the scrutinee syntactically (which gives us `cx.known_valid_scrutinee`), and
-//! then by tracking validity of each column of the matrix (which correspond to places) as we
-//! recurse into subpatterns. That second part is done through [`ValidityConstraint`], most notably
-//! [`ValidityConstraint::specialize`].
-//!
-//! Having said all that, in practice we don't fully follow what's been presented in this section.
-//! Under `exhaustive_patterns`, we allow omitting empty arms even in `!known_valid` places, for
-//! backwards-compatibility until we have a better alternative. Without `exhaustive_patterns`, we
-//! mostly treat empty types as inhabited, except specifically a non-nested `!` or empty enum. In
-//! this specific case we also allow the empty match regardless of place validity, for
-//! backwards-compatibility. Hopefully we can eventually deprecate this.
-//!
-//!
-//!
-//! # Full example
-//!
-//! We illustrate a full run of the algorithm on the following match.
-//!
-//! ```compile_fail,E0004
-//! # struct Pair(Option<u32>, bool);
-//! # fn foo(x: Pair) -> u32 {
-//! match x {
-//!     Pair(Some(0), _) => 1,
-//!     Pair(_, false) => 2,
-//!     Pair(Some(0), false) => 3,
-//! }
-//! # }
-//! ```
-//!
-//! We keep track of the original row for illustration purposes, this is not what the algorithm
-//! actually does (it tracks usefulness as a boolean on each row).
-//!
-//! ```text
-//!  ┐ Patterns:
-//!  │   1. `[Pair(Some(0), _)]`
-//!  │   2. `[Pair(_, false)]`
-//!  │   3. `[Pair(Some(0), false)]`
-//!  │
-//!  │ Specialize with `Pair`:
-//!  ├─┐ Patterns:
-//!  │ │   1. `[Some(0), _]`
-//!  │ │   2. `[_, false]`
-//!  │ │   3. `[Some(0), false]`
-//!  │ │
-//!  │ │ Specialize with `Some`:
-//!  │ ├─┐ Patterns:
-//!  │ │ │   1. `[0, _]`
-//!  │ │ │   2. `[_, false]`
-//!  │ │ │   3. `[0, false]`
-//!  │ │ │
-//!  │ │ │ Specialize with `0`:
-//!  │ │ ├─┐ Patterns:
-//!  │ │ │ │   1. `[_]`
-//!  │ │ │ │   3. `[false]`
-//!  │ │ │ │
-//!  │ │ │ │ Specialize with `true`:
-//!  │ │ │ ├─┐ Patterns:
-//!  │ │ │ │ │   1. `[]`
-//!  │ │ │ │ │
-//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), true)`).
-//!  │ │ │ ├─┘
-//!  │ │ │ │
-//!  │ │ │ │ Specialize with `false`:
-//!  │ │ │ ├─┐ Patterns:
-//!  │ │ │ │ │   1. `[]`
-//!  │ │ │ │ │   3. `[]`
-//!  │ │ │ │ │
-//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), false)`).
-//!  │ │ │ ├─┘
-//!  │ │ ├─┘
-//!  │ │ │
-//!  │ │ │ Specialize with `1..`:
-//!  │ │ ├─┐ Patterns:
-//!  │ │ │ │   2. `[false]`
-//!  │ │ │ │
-//!  │ │ │ │ Specialize with `true`:
-//!  │ │ │ ├─┐ Patterns:
-//!  │ │ │ │ │   // no rows left
-//!  │ │ │ │ │
-//!  │ │ │ │ │ We have found an unmatched value (`Pair(Some(1..), true)`)! This gives us a witness.
-//!  │ │ │ │ │ New witnesses:
-//!  │ │ │ │ │   `[]`
-//!  │ │ │ ├─┘
-//!  │ │ │ │ Unspecialize new witnesses with `true`:
-//!  │ │ │ │   `[true]`
-//!  │ │ │ │
-//!  │ │ │ │ Specialize with `false`:
-//!  │ │ │ ├─┐ Patterns:
-//!  │ │ │ │ │   2. `[]`
-//!  │ │ │ │ │
-//!  │ │ │ │ │ We note arm 2 is useful (by `Pair(Some(1..), false)`).
-//!  │ │ │ ├─┘
-//!  │ │ │ │
-//!  │ │ │ │ Total witnesses for `1..`:
-//!  │ │ │ │   `[true]`
-//!  │ │ ├─┘
-//!  │ │ │ Unspecialize new witnesses with `1..`:
-//!  │ │ │   `[1.., true]`
-//!  │ │ │
-//!  │ │ │ Total witnesses for `Some`:
-//!  │ │ │   `[1.., true]`
-//!  │ ├─┘
-//!  │ │ Unspecialize new witnesses with `Some`:
-//!  │ │   `[Some(1..), true]`
-//!  │ │
-//!  │ │ Specialize with `None`:
-//!  │ ├─┐ Patterns:
-//!  │ │ │   2. `[false]`
-//!  │ │ │
-//!  │ │ │ Specialize with `true`:
-//!  │ │ ├─┐ Patterns:
-//!  │ │ │ │   // no rows left
-//!  │ │ │ │
-//!  │ │ │ │ We have found an unmatched value (`Pair(None, true)`)! This gives us a witness.
-//!  │ │ │ │ New witnesses:
-//!  │ │ │ │   `[]`
-//!  │ │ ├─┘
-//!  │ │ │ Unspecialize new witnesses with `true`:
-//!  │ │ │   `[true]`
-//!  │ │ │
-//!  │ │ │ Specialize with `false`:
-//!  │ │ ├─┐ Patterns:
-//!  │ │ │ │   2. `[]`
-//!  │ │ │ │
-//!  │ │ │ │ We note arm 2 is useful (by `Pair(None, false)`).
-//!  │ │ ├─┘
-//!  │ │ │
-//!  │ │ │ Total witnesses for `None`:
-//!  │ │ │   `[true]`
-//!  │ ├─┘
-//!  │ │ Unspecialize new witnesses with `None`:
-//!  │ │   `[None, true]`
-//!  │ │
-//!  │ │ Total witnesses for `Pair`:
-//!  │ │   `[Some(1..), true]`
-//!  │ │   `[None, true]`
-//!  ├─┘
-//!  │ Unspecialize new witnesses with `Pair`:
-//!  │   `[Pair(Some(1..), true)]`
-//!  │   `[Pair(None, true)]`
-//!  │
-//!  │ Final witnesses:
-//!  │   `[Pair(Some(1..), true)]`
-//!  │   `[Pair(None, true)]`
-//!  ┘
-//! ```
-//!
-//! We conclude:
-//! - Arm 3 is redundant (it was never marked as useful);
-//! - The match is not exhaustive;
-//! - Adding arms with `Pair(Some(1..), true)` and `Pair(None, true)` would make the match exhaustive.
-//!
-//! Note that when we're deep in the algorithm, we don't know what specialization steps got us here.
-//! We can only figure out what our witnesses correspond to by unspecializing back up the stack.
-//!
-//!
-//! # Tests
-//!
-//! Note: tests specific to this file can be found in:
-//!
-//!   - `ui/pattern/usefulness`
-//!   - `ui/or-patterns`
-//!   - `ui/consts/const_in_pattern`
-//!   - `ui/rfc-2008-non-exhaustive`
-//!   - `ui/half-open-range-patterns`
-//!   - probably many others
-//!
-//! I (Nadrieril) prefer to put new tests in `ui/pattern/usefulness` unless there's a specific
-//! reason not to, for example if they crucially depend on a particular feature like `or_patterns`.
-
-use self::ValidityConstraint::*;
-use super::deconstruct_pat::{
-    Constructor, ConstructorSet, DeconstructedPat, IntRange, MaybeInfiniteInt, SplitConstructorSet,
-    WitnessPat,
-};
-use crate::errors::{
-    NonExhaustiveOmittedPattern, NonExhaustiveOmittedPatternLintOnArm, Overlap,
-    OverlappingRangeEndpoints, Uncovered,
-};
-
-use rustc_data_structures::captures::Captures;
-
-use rustc_arena::TypedArena;
-use rustc_data_structures::stack::ensure_sufficient_stack;
-use rustc_hir::def_id::DefId;
-use rustc_hir::HirId;
-use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_session::lint;
-use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
-use rustc_span::{Span, DUMMY_SP};
-
-use smallvec::{smallvec, SmallVec};
-use std::fmt;
-
-pub(crate) struct MatchCheckCtxt<'p, 'tcx> {
-    pub(crate) tcx: TyCtxt<'tcx>,
-    /// The module in which the match occurs. This is necessary for
-    /// checking inhabited-ness of types because whether a type is (visibly)
-    /// inhabited can depend on whether it was defined in the current module or
-    /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
-    /// outside its module and should not be matchable with an empty match statement.
-    pub(crate) module: DefId,
-    pub(crate) param_env: ty::ParamEnv<'tcx>,
-    pub(crate) pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
-    /// Lint level at the match.
-    pub(crate) match_lint_level: HirId,
-    /// The span of the whole match, if applicable.
-    pub(crate) whole_match_span: Option<Span>,
-    /// Span of the scrutinee.
-    pub(crate) scrut_span: Span,
-    /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
-    pub(crate) refutable: bool,
-    /// Whether the data at the scrutinee is known to be valid. This is false if the scrutinee comes
-    /// from a union field, a pointer deref, or a reference deref (pending opsem decisions).
-    pub(crate) known_valid_scrutinee: bool,
-}
-
-impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
-    pub(super) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
-        !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
-    }
-
-    /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
-    pub(super) fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
-        match ty.kind() {
-            ty::Adt(def, ..) => {
-                def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did().is_local()
-            }
-            _ => false,
-        }
-    }
-}
-
-#[derive(Copy, Clone)]
-pub(super) struct PatCtxt<'a, 'p, 'tcx> {
-    pub(super) cx: &'a MatchCheckCtxt<'p, 'tcx>,
-    /// Type of the current column under investigation.
-    pub(super) ty: Ty<'tcx>,
-    /// Whether the current pattern is the whole pattern as found in a match arm, or if it's a
-    /// subpattern.
-    pub(super) is_top_level: bool,
-}
-
-impl<'a, 'p, 'tcx> PatCtxt<'a, 'p, 'tcx> {
-    /// A `PatCtxt` when code other than `is_useful` needs one.
-    fn new_dummy(cx: &'a MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
-        PatCtxt { cx, ty, is_top_level: false }
-    }
-}
-
-impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("PatCtxt").field("ty", &self.ty).finish()
-    }
-}
-
-/// Serves two purposes:
-/// - in a wildcard, tracks whether the wildcard matches only valid values (i.e. is a binding `_a`)
-///     or also invalid values (i.e. is a true `_` pattern).
-/// - in the matrix, track whether a given place (aka column) is known to contain a valid value or
-///     not.
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub(super) enum ValidityConstraint {
-    ValidOnly,
-    MaybeInvalid,
-    /// Option for backwards compatibility: the place is not known to be valid but we allow omitting
-    /// `useful && !reachable` arms anyway.
-    MaybeInvalidButAllowOmittingArms,
-}
-
-impl ValidityConstraint {
-    pub(super) fn from_bool(is_valid_only: bool) -> Self {
-        if is_valid_only { ValidOnly } else { MaybeInvalid }
-    }
-
-    fn allow_omitting_side_effecting_arms(self) -> Self {
-        match self {
-            MaybeInvalid | MaybeInvalidButAllowOmittingArms => MaybeInvalidButAllowOmittingArms,
-            // There are no side-effecting empty arms here, nothing to do.
-            ValidOnly => ValidOnly,
-        }
-    }
-
-    pub(super) fn is_known_valid(self) -> bool {
-        matches!(self, ValidOnly)
-    }
-    pub(super) fn allows_omitting_empty_arms(self) -> bool {
-        matches!(self, ValidOnly | MaybeInvalidButAllowOmittingArms)
-    }
-
-    /// If the place has validity given by `self` and we read that the value at the place has
-    /// constructor `ctor`, this computes what we can assume about the validity of the constructor
-    /// fields.
-    ///
-    /// Pending further opsem decisions, the current behavior is: validity is preserved, except
-    /// inside `&` and union fields where validity is reset to `MaybeInvalid`.
-    pub(super) fn specialize<'tcx>(
-        self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        ctor: &Constructor<'tcx>,
-    ) -> Self {
-        // We preserve validity except when we go inside a reference or a union field.
-        if matches!(ctor, Constructor::Single)
-            && (matches!(pcx.ty.kind(), ty::Ref(..))
-                || matches!(pcx.ty.kind(), ty::Adt(def, ..) if def.is_union()))
-        {
-            // Validity of `x: &T` does not imply validity of `*x: T`.
-            MaybeInvalid
-        } else {
-            self
-        }
-    }
-}
-
-impl fmt::Display for ValidityConstraint {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let s = match self {
-            ValidOnly => "✓",
-            MaybeInvalid | MaybeInvalidButAllowOmittingArms => "?",
-        };
-        write!(f, "{s}")
-    }
-}
-
-/// Represents a pattern-tuple under investigation.
-#[derive(Clone)]
-struct PatStack<'p, 'tcx> {
-    // Rows of len 1 are very common, which is why `SmallVec[_; 2]` works well.
-    pats: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>,
-}
-
-impl<'p, 'tcx> PatStack<'p, 'tcx> {
-    fn from_pattern(pat: &'p DeconstructedPat<'p, 'tcx>) -> Self {
-        PatStack { pats: smallvec![pat] }
-    }
-
-    fn is_empty(&self) -> bool {
-        self.pats.is_empty()
-    }
-
-    fn len(&self) -> usize {
-        self.pats.len()
-    }
-
-    fn head(&self) -> &'p DeconstructedPat<'p, 'tcx> {
-        self.pats[0]
-    }
-
-    fn iter(&self) -> impl Iterator<Item = &DeconstructedPat<'p, 'tcx>> {
-        self.pats.iter().copied()
-    }
-
-    // Recursively expand the first or-pattern into its subpatterns. Only useful if the pattern is
-    // an or-pattern. Panics if `self` is empty.
-    fn expand_or_pat<'a>(&'a self) -> impl Iterator<Item = PatStack<'p, 'tcx>> + Captures<'a> {
-        self.head().flatten_or_pat().into_iter().map(move |pat| {
-            let mut new = self.clone();
-            new.pats[0] = pat;
-            new
-        })
-    }
-
-    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
-    /// Only call if `ctor.is_covered_by(self.head().ctor())` is true.
-    fn pop_head_constructor(
-        &self,
-        pcx: &PatCtxt<'_, 'p, 'tcx>,
-        ctor: &Constructor<'tcx>,
-    ) -> PatStack<'p, 'tcx> {
-        // We pop the head pattern and push the new fields extracted from the arguments of
-        // `self.head()`.
-        let mut new_pats = self.head().specialize(pcx, ctor);
-        new_pats.extend_from_slice(&self.pats[1..]);
-        PatStack { pats: new_pats }
-    }
-}
-
-impl<'p, 'tcx> fmt::Debug for PatStack<'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // We pretty-print similarly to the `Debug` impl of `Matrix`.
-        write!(f, "+")?;
-        for pat in self.iter() {
-            write!(f, " {pat:?} +")?;
-        }
-        Ok(())
-    }
-}
-
-/// A row of the matrix.
-#[derive(Clone)]
-struct MatrixRow<'p, 'tcx> {
-    // The patterns in the row.
-    pats: PatStack<'p, 'tcx>,
-    /// Whether the original arm had a guard. This is inherited when specializing.
-    is_under_guard: bool,
-    /// When we specialize, we remember which row of the original matrix produced a given row of the
-    /// specialized matrix. When we unspecialize, we use this to propagate usefulness back up the
-    /// callstack.
-    parent_row: usize,
-    /// False when the matrix is just built. This is set to `true` by
-    /// [`compute_exhaustiveness_and_usefulness`] if the arm is found to be useful.
-    /// This is reset to `false` when specializing.
-    useful: bool,
-}
-
-impl<'p, 'tcx> MatrixRow<'p, 'tcx> {
-    fn is_empty(&self) -> bool {
-        self.pats.is_empty()
-    }
-
-    fn len(&self) -> usize {
-        self.pats.len()
-    }
-
-    fn head(&self) -> &'p DeconstructedPat<'p, 'tcx> {
-        self.pats.head()
-    }
-
-    fn iter(&self) -> impl Iterator<Item = &DeconstructedPat<'p, 'tcx>> {
-        self.pats.iter()
-    }
-
-    // Recursively expand the first or-pattern into its subpatterns. Only useful if the pattern is
-    // an or-pattern. Panics if `self` is empty.
-    fn expand_or_pat<'a>(&'a self) -> impl Iterator<Item = MatrixRow<'p, 'tcx>> + Captures<'a> {
-        self.pats.expand_or_pat().map(|patstack| MatrixRow {
-            pats: patstack,
-            parent_row: self.parent_row,
-            is_under_guard: self.is_under_guard,
-            useful: false,
-        })
-    }
-
-    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
-    /// Only call if `ctor.is_covered_by(self.head().ctor())` is true.
-    fn pop_head_constructor(
-        &self,
-        pcx: &PatCtxt<'_, 'p, 'tcx>,
-        ctor: &Constructor<'tcx>,
-        parent_row: usize,
-    ) -> MatrixRow<'p, 'tcx> {
-        MatrixRow {
-            pats: self.pats.pop_head_constructor(pcx, ctor),
-            parent_row,
-            is_under_guard: self.is_under_guard,
-            useful: false,
-        }
-    }
-}
-
-impl<'p, 'tcx> fmt::Debug for MatrixRow<'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.pats.fmt(f)
-    }
-}
-
-/// A 2D matrix. Represents a list of pattern-tuples under investigation.
-///
-/// Invariant: each row must have the same length, and each column must have the same type.
-///
-/// Invariant: the first column must not contain or-patterns. This is handled by
-/// [`Matrix::expand_and_push`].
-///
-/// In fact each column corresponds to a place inside the scrutinee of the match. E.g. after
-/// specializing `(,)` and `Some` on a pattern of type `(Option<u32>, bool)`, the first column of
-/// the matrix will correspond to `scrutinee.0.Some.0` and the second column to `scrutinee.1`.
-#[derive(Clone)]
-struct Matrix<'p, 'tcx> {
-    /// Vector of rows. The rows must form a rectangular 2D array. Moreover, all the patterns of
-    /// each column must have the same type. Each column corresponds to a place within the
-    /// scrutinee.
-    rows: Vec<MatrixRow<'p, 'tcx>>,
-    /// Stores an extra fictitious row full of wildcards. Mostly used to keep track of the type of
-    /// each column. This must obey the same invariants as the real rows.
-    wildcard_row: PatStack<'p, 'tcx>,
-    /// Track for each column/place whether it contains a known valid value.
-    place_validity: SmallVec<[ValidityConstraint; 2]>,
-}
-
-impl<'p, 'tcx> Matrix<'p, 'tcx> {
-    /// Pushes a new row to the matrix. If the row starts with an or-pattern, this recursively
-    /// expands it. Internal method, prefer [`Matrix::new`].
-    fn expand_and_push(&mut self, row: MatrixRow<'p, 'tcx>) {
-        if !row.is_empty() && row.head().is_or_pat() {
-            // Expand nested or-patterns.
-            for new_row in row.expand_or_pat() {
-                self.rows.push(new_row);
-            }
-        } else {
-            self.rows.push(row);
-        }
-    }
-
-    /// Build a new matrix from an iterator of `MatchArm`s.
-    fn new<'a>(
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        arms: &[MatchArm<'p, 'tcx>],
-        scrut_ty: Ty<'tcx>,
-        scrut_validity: ValidityConstraint,
-    ) -> Self
-    where
-        'p: 'a,
-    {
-        let wild_pattern = cx.pattern_arena.alloc(DeconstructedPat::wildcard(scrut_ty, DUMMY_SP));
-        let wildcard_row = PatStack::from_pattern(wild_pattern);
-        let mut matrix = Matrix {
-            rows: Vec::with_capacity(arms.len()),
-            wildcard_row,
-            place_validity: smallvec![scrut_validity],
-        };
-        for (row_id, arm) in arms.iter().enumerate() {
-            let v = MatrixRow {
-                pats: PatStack::from_pattern(arm.pat),
-                parent_row: row_id, // dummy, we won't read it
-                is_under_guard: arm.has_guard,
-                useful: false,
-            };
-            matrix.expand_and_push(v);
-        }
-        matrix
-    }
-
-    fn head_ty(&self) -> Option<Ty<'tcx>> {
-        if self.column_count() == 0 {
-            return None;
-        }
-
-        let mut ty = self.wildcard_row.head().ty();
-        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
-        // version. Otherwise we could encounter constructors for the revealed type and crash.
-        let is_opaque = |ty: Ty<'tcx>| matches!(ty.kind(), ty::Alias(ty::Opaque, ..));
-        if is_opaque(ty) {
-            for pat in self.heads() {
-                let pat_ty = pat.ty();
-                if !is_opaque(pat_ty) {
-                    ty = pat_ty;
-                    break;
-                }
-            }
-        }
-        Some(ty)
-    }
-    fn column_count(&self) -> usize {
-        self.wildcard_row.len()
-    }
-
-    fn rows<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'a MatrixRow<'p, 'tcx>> + Clone + DoubleEndedIterator + ExactSizeIterator
-    {
-        self.rows.iter()
-    }
-    fn rows_mut<'a>(
-        &'a mut self,
-    ) -> impl Iterator<Item = &'a mut MatrixRow<'p, 'tcx>> + DoubleEndedIterator + ExactSizeIterator
-    {
-        self.rows.iter_mut()
-    }
-
-    /// Iterate over the first pattern of each row.
-    fn heads<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Clone + Captures<'a> {
-        self.rows().map(|r| r.head())
-    }
-
-    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
-    fn specialize_constructor(
-        &self,
-        pcx: &PatCtxt<'_, 'p, 'tcx>,
-        ctor: &Constructor<'tcx>,
-    ) -> Matrix<'p, 'tcx> {
-        let wildcard_row = self.wildcard_row.pop_head_constructor(pcx, ctor);
-        let new_validity = self.place_validity[0].specialize(pcx, ctor);
-        let new_place_validity = std::iter::repeat(new_validity)
-            .take(ctor.arity(pcx))
-            .chain(self.place_validity[1..].iter().copied())
-            .collect();
-        let mut matrix =
-            Matrix { rows: Vec::new(), wildcard_row, place_validity: new_place_validity };
-        for (i, row) in self.rows().enumerate() {
-            if ctor.is_covered_by(pcx, row.head().ctor()) {
-                let new_row = row.pop_head_constructor(pcx, ctor, i);
-                matrix.expand_and_push(new_row);
-            }
-        }
-        matrix
-    }
-}
-
-/// Pretty-printer for matrices of patterns, example:
-///
-/// ```text
-/// + _     + []                +
-/// + true  + [First]           +
-/// + true  + [Second(true)]    +
-/// + false + [_]               +
-/// + _     + [_, _, tail @ ..] +
-/// | ✓     | ?                 | // column validity
-/// ```
-impl<'p, 'tcx> fmt::Debug for Matrix<'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "\n")?;
-
-        let mut pretty_printed_matrix: Vec<Vec<String>> = self
-            .rows
-            .iter()
-            .map(|row| row.iter().map(|pat| format!("{pat:?}")).collect())
-            .collect();
-        pretty_printed_matrix
-            .push(self.place_validity.iter().map(|validity| format!("{validity}")).collect());
-
-        let column_count = self.column_count();
-        assert!(self.rows.iter().all(|row| row.len() == column_count));
-        assert!(self.place_validity.len() == column_count);
-        let column_widths: Vec<usize> = (0..column_count)
-            .map(|col| pretty_printed_matrix.iter().map(|row| row[col].len()).max().unwrap_or(0))
-            .collect();
-
-        for (row_i, row) in pretty_printed_matrix.into_iter().enumerate() {
-            let is_validity_row = row_i == self.rows.len();
-            let sep = if is_validity_row { "|" } else { "+" };
-            write!(f, "{sep}")?;
-            for (column, pat_str) in row.into_iter().enumerate() {
-                write!(f, " ")?;
-                write!(f, "{:1$}", pat_str, column_widths[column])?;
-                write!(f, " {sep}")?;
-            }
-            if is_validity_row {
-                write!(f, " // column validity")?;
-            }
-            write!(f, "\n")?;
-        }
-        Ok(())
-    }
-}
-
-/// A witness-tuple of non-exhaustiveness for error reporting, represented as a list of patterns (in
-/// reverse order of construction).
-///
-/// This mirrors `PatStack`: they function similarly, except `PatStack` contains user patterns we
-/// are inspecting, and `WitnessStack` contains witnesses we are constructing.
-/// FIXME(Nadrieril): use the same order of patterns for both.
-///
-/// A `WitnessStack` should have the same types and length as the `PatStack`s we are inspecting
-/// (except we store the patterns in reverse order). The same way `PatStack` starts with length 1,
-/// at the end of the algorithm this will have length 1. In the middle of the algorithm, it can
-/// contain multiple patterns.
-///
-/// For example, if we are constructing a witness for the match against
-///
-/// ```compile_fail,E0004
-/// struct Pair(Option<(u32, u32)>, bool);
-/// # fn foo(p: Pair) {
-/// match p {
-///    Pair(None, _) => {}
-///    Pair(_, false) => {}
-/// }
-/// # }
-/// ```
-///
-/// We'll perform the following steps (among others):
-/// ```text
-/// - Start with a matrix representing the match
-///     `PatStack(vec![Pair(None, _)])`
-///     `PatStack(vec![Pair(_, false)])`
-/// - Specialize with `Pair`
-///     `PatStack(vec![None, _])`
-///     `PatStack(vec![_, false])`
-/// - Specialize with `Some`
-///     `PatStack(vec![_, false])`
-/// - Specialize with `_`
-///     `PatStack(vec![false])`
-/// - Specialize with `true`
-///     // no patstacks left
-/// - This is a non-exhaustive match: we have the empty witness stack as a witness.
-///     `WitnessStack(vec![])`
-/// - Apply `true`
-///     `WitnessStack(vec![true])`
-/// - Apply `_`
-///     `WitnessStack(vec![true, _])`
-/// - Apply `Some`
-///     `WitnessStack(vec![true, Some(_)])`
-/// - Apply `Pair`
-///     `WitnessStack(vec![Pair(Some(_), true)])`
-/// ```
-///
-/// The final `Pair(Some(_), true)` is then the resulting witness.
-///
-/// See the top of the file for more detailed explanations and examples.
-#[derive(Debug, Clone)]
-pub(crate) struct WitnessStack<'tcx>(Vec<WitnessPat<'tcx>>);
-
-impl<'tcx> WitnessStack<'tcx> {
-    /// Asserts that the witness contains a single pattern, and returns it.
-    fn single_pattern(self) -> WitnessPat<'tcx> {
-        assert_eq!(self.0.len(), 1);
-        self.0.into_iter().next().unwrap()
-    }
-
-    /// Reverses specialization by the `Missing` constructor by pushing a whole new pattern.
-    fn push_pattern(&mut self, pat: WitnessPat<'tcx>) {
-        self.0.push(pat);
-    }
-
-    /// Reverses specialization. Given a witness obtained after specialization, this constructs a
-    /// new witness valid for before specialization. See the section on `unspecialize` at the top of
-    /// the file.
-    ///
-    /// Examples:
-    /// ```text
-    /// ctor: tuple of 2 elements
-    /// pats: [false, "foo", _, true]
-    /// result: [(false, "foo"), _, true]
-    ///
-    /// ctor: Enum::Variant { a: (bool, &'static str), b: usize}
-    /// pats: [(false, "foo"), _, true]
-    /// result: [Enum::Variant { a: (false, "foo"), b: _ }, true]
-    /// ```
-    fn apply_constructor(&mut self, pcx: &PatCtxt<'_, '_, 'tcx>, ctor: &Constructor<'tcx>) {
-        let len = self.0.len();
-        let arity = ctor.arity(pcx);
-        let fields = self.0.drain((len - arity)..).rev().collect();
-        let pat = WitnessPat::new(ctor.clone(), fields, pcx.ty);
-        self.0.push(pat);
-    }
-}
-
-/// Represents a set of pattern-tuples that are witnesses of non-exhaustiveness for error
-/// reporting. This has similar invariants as `Matrix` does.
-///
-/// The `WitnessMatrix` returned by [`compute_exhaustiveness_and_usefulness`] obeys the invariant
-/// that the union of the input `Matrix` and the output `WitnessMatrix` together matches the type
-/// exhaustively.
-///
-/// Just as the `Matrix` starts with a single column, by the end of the algorithm, this has a single
-/// column, which contains the patterns that are missing for the match to be exhaustive.
-#[derive(Debug, Clone)]
-pub struct WitnessMatrix<'tcx>(Vec<WitnessStack<'tcx>>);
-
-impl<'tcx> WitnessMatrix<'tcx> {
-    /// New matrix with no witnesses.
-    fn empty() -> Self {
-        WitnessMatrix(vec![])
-    }
-    /// New matrix with one `()` witness, i.e. with no columns.
-    fn unit_witness() -> Self {
-        WitnessMatrix(vec![WitnessStack(vec![])])
-    }
-
-    /// Whether this has any witnesses.
-    fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-    /// Asserts that there is a single column and returns the patterns in it.
-    fn single_column(self) -> Vec<WitnessPat<'tcx>> {
-        self.0.into_iter().map(|w| w.single_pattern()).collect()
-    }
-
-    /// Reverses specialization by the `Missing` constructor by pushing a whole new pattern.
-    fn push_pattern(&mut self, pat: WitnessPat<'tcx>) {
-        for witness in self.0.iter_mut() {
-            witness.push_pattern(pat.clone())
-        }
-    }
-
-    /// Reverses specialization by `ctor`. See the section on `unspecialize` at the top of the file.
-    fn apply_constructor(
-        &mut self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        missing_ctors: &[Constructor<'tcx>],
-        ctor: &Constructor<'tcx>,
-        report_individual_missing_ctors: bool,
-    ) {
-        if self.is_empty() {
-            return;
-        }
-        if matches!(ctor, Constructor::Missing) {
-            // We got the special `Missing` constructor that stands for the constructors not present
-            // in the match.
-            if !report_individual_missing_ctors {
-                // Report `_` as missing.
-                let pat = WitnessPat::wild_from_ctor(pcx, Constructor::Wildcard);
-                self.push_pattern(pat);
-            } else if missing_ctors.iter().any(|c| c.is_non_exhaustive()) {
-                // We need to report a `_` anyway, so listing other constructors would be redundant.
-                // `NonExhaustive` is displayed as `_` just like `Wildcard`, but it will be picked
-                // up by diagnostics to add a note about why `_` is required here.
-                let pat = WitnessPat::wild_from_ctor(pcx, Constructor::NonExhaustive);
-                self.push_pattern(pat);
-            } else {
-                // For each missing constructor `c`, we add a `c(_, _, _)` witness appropriately
-                // filled with wildcards.
-                let mut ret = Self::empty();
-                for ctor in missing_ctors {
-                    let pat = WitnessPat::wild_from_ctor(pcx, ctor.clone());
-                    // Clone `self` and add `c(_, _, _)` to each of its witnesses.
-                    let mut wit_matrix = self.clone();
-                    wit_matrix.push_pattern(pat);
-                    ret.extend(wit_matrix);
-                }
-                *self = ret;
-            }
-        } else {
-            // Any other constructor we unspecialize as expected.
-            for witness in self.0.iter_mut() {
-                witness.apply_constructor(pcx, ctor)
-            }
-        }
-    }
-
-    /// Merges the witnesses of two matrices. Their column types must match.
-    fn extend(&mut self, other: Self) {
-        self.0.extend(other.0)
-    }
-}
-
-/// The core of the algorithm.
-///
-/// This recursively computes witnesses of the non-exhaustiveness of `matrix` (if any). Also tracks
-/// usefulness of each row in the matrix (in `row.useful`). We track usefulness of each
-/// subpattern using interior mutability in `DeconstructedPat`.
-///
-/// The input `Matrix` and the output `WitnessMatrix` together match the type exhaustively.
-///
-/// The key steps are:
-/// - specialization, where we dig into the rows that have a specific constructor and call ourselves
-///     recursively;
-/// - unspecialization, where we lift the results from the previous step into results for this step
-///     (using `apply_constructor` and by updating `row.useful` for each parent row).
-/// This is all explained at the top of the file.
-#[instrument(level = "debug", skip(cx, is_top_level), ret)]
-fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    matrix: &mut Matrix<'p, 'tcx>,
-    is_top_level: bool,
-) -> WitnessMatrix<'tcx> {
-    debug_assert!(matrix.rows().all(|r| r.len() == matrix.column_count()));
-
-    let Some(ty) = matrix.head_ty() else {
-        // The base case: there are no columns in the matrix. We are morally pattern-matching on ().
-        // A row is useful iff it has no (unguarded) rows above it.
-        for row in matrix.rows_mut() {
-            // All rows are useful until they're not.
-            row.useful = true;
-            // When there's an unguarded row, the match is exhaustive and any subsequent row is not
-            // useful.
-            if !row.is_under_guard {
-                return WitnessMatrix::empty();
-            }
-        }
-        // No (unguarded) rows, so the match is not exhaustive. We return a new witness.
-        return WitnessMatrix::unit_witness();
-    };
-
-    debug!("ty: {ty:?}");
-    let pcx = &PatCtxt { cx, ty, is_top_level };
-
-    // Whether the place/column we are inspecting is known to contain valid data.
-    let place_validity = matrix.place_validity[0];
-    // For backwards compability we allow omitting some empty arms that we ideally shouldn't.
-    let place_validity = place_validity.allow_omitting_side_effecting_arms();
-
-    // Analyze the constructors present in this column.
-    let ctors = matrix.heads().map(|p| p.ctor());
-    let split_set = ConstructorSet::for_ty(cx, ty).split(pcx, ctors);
-    let all_missing = split_set.present.is_empty();
-
-    // Build the set of constructors we will specialize with. It must cover the whole type.
-    let mut split_ctors = split_set.present;
-    if !split_set.missing.is_empty() {
-        // We need to iterate over a full set of constructors, so we add `Missing` to represent the
-        // missing ones. This is explained under "Constructor Splitting" at the top of this file.
-        split_ctors.push(Constructor::Missing);
-    } else if !split_set.missing_empty.is_empty() && !place_validity.is_known_valid() {
-        // The missing empty constructors are reachable if the place can contain invalid data.
-        split_ctors.push(Constructor::Missing);
-    }
-
-    // Decide what constructors to report.
-    let always_report_all = is_top_level && !IntRange::is_integral(pcx.ty);
-    // Whether we should report "Enum::A and Enum::C are missing" or "_ is missing".
-    let report_individual_missing_ctors = always_report_all || !all_missing;
-    // Which constructors are considered missing. We ensure that `!missing_ctors.is_empty() =>
-    // split_ctors.contains(Missing)`. The converse usually holds except in the
-    // `MaybeInvalidButAllowOmittingArms` backwards-compatibility case.
-    let mut missing_ctors = split_set.missing;
-    if !place_validity.allows_omitting_empty_arms() {
-        missing_ctors.extend(split_set.missing_empty);
-    }
-
-    let mut ret = WitnessMatrix::empty();
-    for ctor in split_ctors {
-        debug!("specialize({:?})", ctor);
-        // Dig into rows that match `ctor`.
-        let mut spec_matrix = matrix.specialize_constructor(pcx, &ctor);
-        let mut witnesses = ensure_sufficient_stack(|| {
-            compute_exhaustiveness_and_usefulness(cx, &mut spec_matrix, false)
-        });
-
-        let counts_for_exhaustiveness = match ctor {
-            Constructor::Missing => !missing_ctors.is_empty(),
-            // If there are missing constructors we'll report those instead. Since `Missing` matches
-            // only the wildcard rows, it matches fewer rows than this constructor, and is therefore
-            // guaranteed to result in the same or more witnesses. So skipping this does not
-            // jeopardize correctness.
-            _ => missing_ctors.is_empty(),
-        };
-        if counts_for_exhaustiveness {
-            // Transform witnesses for `spec_matrix` into witnesses for `matrix`.
-            witnesses.apply_constructor(
-                pcx,
-                &missing_ctors,
-                &ctor,
-                report_individual_missing_ctors,
-            );
-            // Accumulate the found witnesses.
-            ret.extend(witnesses);
-        }
-
-        // A parent row is useful if any of its children is.
-        for child_row in spec_matrix.rows() {
-            let parent_row = &mut matrix.rows[child_row.parent_row];
-            parent_row.useful = parent_row.useful || child_row.useful;
-        }
-    }
-
-    // Record usefulness in the patterns.
-    for row in matrix.rows() {
-        if row.useful {
-            row.head().set_useful();
-        }
-    }
-
-    ret
-}
-
-/// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
-/// inspect the same subvalue/place".
-/// This is used to traverse patterns column-by-column for lints. Despite similarities with
-/// [`compute_exhaustiveness_and_usefulness`], this does a different traversal. Notably this is
-/// linear in the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case
-/// exponential (exhaustiveness is NP-complete). The core difference is that we treat sub-columns
-/// separately.
-///
-/// This must not contain an or-pattern. `specialize` takes care to expand them.
-///
-/// This is not used in the main algorithm; only in lints.
-#[derive(Debug)]
-struct PatternColumn<'p, 'tcx> {
-    patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>,
-}
-
-impl<'p, 'tcx> PatternColumn<'p, 'tcx> {
-    fn new(patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>) -> Self {
-        Self { patterns }
-    }
-
-    fn is_empty(&self) -> bool {
-        self.patterns.is_empty()
-    }
-    fn head_ty(&self) -> Option<Ty<'tcx>> {
-        if self.patterns.len() == 0 {
-            return None;
-        }
-        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
-        // version. Otherwise we could encounter constructors for the revealed type and crash.
-        let is_opaque = |ty: Ty<'tcx>| matches!(ty.kind(), ty::Alias(ty::Opaque, ..));
-        let first_ty = self.patterns[0].ty();
-        if is_opaque(first_ty) {
-            for pat in &self.patterns {
-                let ty = pat.ty();
-                if !is_opaque(ty) {
-                    return Some(ty);
-                }
-            }
-        }
-        Some(first_ty)
-    }
-
-    /// Do constructor splitting on the constructors of the column.
-    fn analyze_ctors(&self, pcx: &PatCtxt<'_, 'p, 'tcx>) -> SplitConstructorSet<'tcx> {
-        let column_ctors = self.patterns.iter().map(|p| p.ctor());
-        ConstructorSet::for_ty(pcx.cx, pcx.ty).split(pcx, column_ctors)
-    }
-
-    fn iter<'a>(&'a self) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.patterns.iter().copied()
-    }
-
-    /// Does specialization: given a constructor, this takes the patterns from the column that match
-    /// the constructor, and outputs their fields.
-    /// This returns one column per field of the constructor. They usually all have the same length
-    /// (the number of patterns in `self` that matched `ctor`), except that we expand or-patterns
-    /// which may change the lengths.
-    fn specialize(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>) -> Vec<Self> {
-        let arity = ctor.arity(pcx);
-        if arity == 0 {
-            return Vec::new();
-        }
-
-        // We specialize the column by `ctor`. This gives us `arity`-many columns of patterns. These
-        // columns may have different lengths in the presence of or-patterns (this is why we can't
-        // reuse `Matrix`).
-        let mut specialized_columns: Vec<_> =
-            (0..arity).map(|_| Self { patterns: Vec::new() }).collect();
-        let relevant_patterns =
-            self.patterns.iter().filter(|pat| ctor.is_covered_by(pcx, pat.ctor()));
-        for pat in relevant_patterns {
-            let specialized = pat.specialize(pcx, ctor);
-            for (subpat, column) in specialized.iter().zip(&mut specialized_columns) {
-                if subpat.is_or_pat() {
-                    column.patterns.extend(subpat.flatten_or_pat())
-                } else {
-                    column.patterns.push(subpat)
-                }
-            }
-        }
-
-        assert!(
-            !specialized_columns[0].is_empty(),
-            "ctor {ctor:?} was listed as present but isn't;
-            there is an inconsistency between `Constructor::is_covered_by` and `ConstructorSet::split`"
-        );
-        specialized_columns
-    }
-}
-
-/// Traverse the patterns to collect any variants of a non_exhaustive enum that fail to be mentioned
-/// in a given column.
-#[instrument(level = "debug", skip(cx), ret)]
-fn collect_nonexhaustive_missing_variants<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    column: &PatternColumn<'p, 'tcx>,
-) -> Vec<WitnessPat<'tcx>> {
-    let Some(ty) = column.head_ty() else {
-        return Vec::new();
-    };
-    let pcx = &PatCtxt::new_dummy(cx, ty);
-
-    let set = column.analyze_ctors(pcx);
-    if set.present.is_empty() {
-        // We can't consistently handle the case where no constructors are present (since this would
-        // require digging deep through any type in case there's a non_exhaustive enum somewhere),
-        // so for consistency we refuse to handle the top-level case, where we could handle it.
-        return vec![];
-    }
-
-    let mut witnesses = Vec::new();
-    if cx.is_foreign_non_exhaustive_enum(ty) {
-        witnesses.extend(
-            set.missing
-                .into_iter()
-                // This will list missing visible variants.
-                .filter(|c| !matches!(c, Constructor::Hidden | Constructor::NonExhaustive))
-                .map(|missing_ctor| WitnessPat::wild_from_ctor(pcx, missing_ctor)),
-        )
-    }
-
-    // Recurse into the fields.
-    for ctor in set.present {
-        let specialized_columns = column.specialize(pcx, &ctor);
-        let wild_pat = WitnessPat::wild_from_ctor(pcx, ctor);
-        for (i, col_i) in specialized_columns.iter().enumerate() {
-            // Compute witnesses for each column.
-            let wits_for_col_i = collect_nonexhaustive_missing_variants(cx, col_i);
-            // For each witness, we build a new pattern in the shape of `ctor(_, _, wit, _, _)`,
-            // adding enough wildcards to match `arity`.
-            for wit in wits_for_col_i {
-                let mut pat = wild_pat.clone();
-                pat.fields[i] = wit;
-                witnesses.push(pat);
-            }
-        }
-    }
-    witnesses
-}
-
-/// Traverse the patterns to warn the user about ranges that overlap on their endpoints.
-#[instrument(level = "debug", skip(cx))]
-fn lint_overlapping_range_endpoints<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    column: &PatternColumn<'p, 'tcx>,
-) {
-    let Some(ty) = column.head_ty() else {
-        return;
-    };
-    let pcx = &PatCtxt::new_dummy(cx, ty);
-
-    let set = column.analyze_ctors(pcx);
-
-    if IntRange::is_integral(ty) {
-        let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
-            let overlap_as_pat = overlap.to_diagnostic_pat(ty, cx.tcx);
-            let overlaps: Vec<_> = overlapped_spans
-                .iter()
-                .copied()
-                .map(|span| Overlap { range: overlap_as_pat.clone(), span })
-                .collect();
-            cx.tcx.emit_spanned_lint(
-                lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
-                cx.match_lint_level,
-                this_span,
-                OverlappingRangeEndpoints { overlap: overlaps, range: this_span },
-            );
-        };
-
-        // If two ranges overlapped, the split set will contain their intersection as a singleton.
-        let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
-        for overlap_range in split_int_ranges.clone() {
-            if overlap_range.is_singleton() {
-                let overlap: MaybeInfiniteInt = overlap_range.lo;
-                // Ranges that look like `lo..=overlap`.
-                let mut prefixes: SmallVec<[_; 1]> = Default::default();
-                // Ranges that look like `overlap..=hi`.
-                let mut suffixes: SmallVec<[_; 1]> = Default::default();
-                // Iterate on patterns that contained `overlap`.
-                for pat in column.iter() {
-                    let this_span = pat.span();
-                    let Constructor::IntRange(this_range) = pat.ctor() else { continue };
-                    if this_range.is_singleton() {
-                        // Don't lint when one of the ranges is a singleton.
-                        continue;
-                    }
-                    if this_range.lo == overlap {
-                        // `this_range` looks like `overlap..=this_range.hi`; it overlaps with any
-                        // ranges that look like `lo..=overlap`.
-                        if !prefixes.is_empty() {
-                            emit_lint(overlap_range, this_span, &prefixes);
-                        }
-                        suffixes.push(this_span)
-                    } else if this_range.hi == overlap.plus_one() {
-                        // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
-                        // ranges that look like `overlap..=hi`.
-                        if !suffixes.is_empty() {
-                            emit_lint(overlap_range, this_span, &suffixes);
-                        }
-                        prefixes.push(this_span)
-                    }
-                }
-            }
-        }
-    } else {
-        // Recurse into the fields.
-        for ctor in set.present {
-            for col in column.specialize(pcx, &ctor) {
-                lint_overlapping_range_endpoints(cx, &col);
-            }
-        }
-    }
-}
-
-/// The arm of a match expression.
-#[derive(Clone, Copy, Debug)]
-pub(crate) struct MatchArm<'p, 'tcx> {
-    /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`.
-    pub(crate) pat: &'p DeconstructedPat<'p, 'tcx>,
-    pub(crate) hir_id: HirId,
-    pub(crate) has_guard: bool,
-}
-
-/// Indicates whether or not a given arm is useful.
-#[derive(Clone, Debug)]
-pub(crate) enum Usefulness {
-    /// The arm is useful. This additionally carries a set of or-pattern branches that have been
-    /// found to be redundant despite the overall arm being useful. Used only in the presence of
-    /// or-patterns, otherwise it stays empty.
-    Useful(Vec<Span>),
-    /// The arm is redundant and can be removed without changing the behavior of the match
-    /// expression.
-    Redundant,
-}
-
-/// The output of checking a match for exhaustiveness and arm usefulness.
-pub(crate) struct UsefulnessReport<'p, 'tcx> {
-    /// For each arm of the input, whether that arm is useful after the arms above it.
-    pub(crate) arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Usefulness)>,
-    /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of
-    /// exhaustiveness.
-    pub(crate) non_exhaustiveness_witnesses: Vec<WitnessPat<'tcx>>,
-}
-
-/// The entrypoint for this file. Computes whether a match is exhaustive and which of its arms are
-/// useful.
-#[instrument(skip(cx, arms), level = "debug")]
-pub(crate) fn compute_match_usefulness<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    arms: &[MatchArm<'p, 'tcx>],
-    scrut_ty: Ty<'tcx>,
-) -> UsefulnessReport<'p, 'tcx> {
-    let scrut_validity = ValidityConstraint::from_bool(cx.known_valid_scrutinee);
-    let mut matrix = Matrix::new(cx, arms, scrut_ty, scrut_validity);
-    let non_exhaustiveness_witnesses = compute_exhaustiveness_and_usefulness(cx, &mut matrix, true);
-
-    let non_exhaustiveness_witnesses: Vec<_> = non_exhaustiveness_witnesses.single_column();
-    let arm_usefulness: Vec<_> = arms
-        .iter()
-        .copied()
-        .map(|arm| {
-            debug!(?arm);
-            // We warn when a pattern is not useful.
-            let usefulness = if arm.pat.is_useful() {
-                Usefulness::Useful(arm.pat.redundant_spans())
-            } else {
-                Usefulness::Redundant
-            };
-            (arm, usefulness)
-        })
-        .collect();
-    let report = UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses };
-
-    let pat_column = PatternColumn::new(matrix.heads().collect());
-    // Lint on ranges that overlap on their endpoints, which is likely a mistake.
-    lint_overlapping_range_endpoints(cx, &pat_column);
-
-    // Run the non_exhaustive_omitted_patterns lint. Only run on refutable patterns to avoid hitting
-    // `if let`s. Only run if the match is exhaustive otherwise the error is redundant.
-    if cx.refutable && report.non_exhaustiveness_witnesses.is_empty() {
-        if !matches!(
-            cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, cx.match_lint_level).0,
-            rustc_session::lint::Level::Allow
-        ) {
-            let witnesses = collect_nonexhaustive_missing_variants(cx, &pat_column);
-            if !witnesses.is_empty() {
-                // Report that a match of a `non_exhaustive` enum marked with `non_exhaustive_omitted_patterns`
-                // is not exhaustive enough.
-                //
-                // NB: The partner lint for structs lives in `compiler/rustc_hir_analysis/src/check/pat.rs`.
-                cx.tcx.emit_spanned_lint(
-                    NON_EXHAUSTIVE_OMITTED_PATTERNS,
-                    cx.match_lint_level,
-                    cx.scrut_span,
-                    NonExhaustiveOmittedPattern {
-                        scrut_ty,
-                        uncovered: Uncovered::new(cx.scrut_span, cx, witnesses),
-                    },
-                );
-            }
-        } else {
-            // We used to allow putting the `#[allow(non_exhaustive_omitted_patterns)]` on a match
-            // arm. This no longer makes sense so we warn users, to avoid silently breaking their
-            // usage of the lint.
-            for arm in arms {
-                let (lint_level, lint_level_source) =
-                    cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, arm.hir_id);
-                if !matches!(lint_level, rustc_session::lint::Level::Allow) {
-                    let decorator = NonExhaustiveOmittedPatternLintOnArm {
-                        lint_span: lint_level_source.span(),
-                        suggest_lint_on_match: cx.whole_match_span.map(|span| span.shrink_to_lo()),
-                        lint_level: lint_level.as_str(),
-                        lint_name: "non_exhaustive_omitted_patterns",
-                    };
-
-                    use rustc_errors::DecorateLint;
-                    let mut err = cx.tcx.sess.struct_span_warn(arm.pat.span(), "");
-                    err.set_primary_message(decorator.msg());
-                    decorator.decorate_lint(&mut err);
-                    err.emit();
-                }
-            }
-        }
-    }
-
-    report
-}
diff --git a/compiler/rustc_pattern_analysis/Cargo.toml b/compiler/rustc_pattern_analysis/Cargo.toml
new file mode 100644
index 00000000000..0639944a45c
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "rustc_pattern_analysis"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+# tidy-alphabetical-start
+rustc_apfloat = "0.2.0"
+rustc_arena = { path = "../rustc_arena" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_pattern_analysis/messages.ftl b/compiler/rustc_pattern_analysis/messages.ftl
new file mode 100644
index 00000000000..827928f97d7
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/messages.ftl
@@ -0,0 +1,19 @@
+pattern_analysis_non_exhaustive_omitted_pattern = some variants are not matched explicitly
+    .help = ensure that all variants are matched explicitly by adding the suggested match arms
+    .note = the matched value is of type `{$scrut_ty}` and the `non_exhaustive_omitted_patterns` attribute was found
+
+pattern_analysis_non_exhaustive_omitted_pattern_lint_on_arm = the lint level must be set on the whole match
+    .help = it no longer has any effect to set the lint level on an individual match arm
+    .label = remove this attribute
+    .suggestion = set the lint level on the whole match
+
+pattern_analysis_overlapping_range_endpoints = multiple patterns overlap on their endpoints
+    .label = ... with this range
+    .note = you likely meant to write mutually exclusive ranges
+
+pattern_analysis_uncovered = {$count ->
+        [1] pattern `{$witness_1}`
+        [2] patterns `{$witness_1}` and `{$witness_2}`
+        [3] patterns `{$witness_1}`, `{$witness_2}` and `{$witness_3}`
+        *[other] patterns `{$witness_1}`, `{$witness_2}`, `{$witness_3}` and {$remainder} more
+    } not covered
diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
new file mode 100644
index 00000000000..4c12cb3b029
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -0,0 +1,1241 @@
+//! As explained in [`super::usefulness`], values and patterns are made from constructors applied to
+//! fields. This file defines a `Constructor` enum and various operations to manipulate them.
+//!
+//! There are two important bits of core logic in this file: constructor inclusion and constructor
+//! splitting. Constructor inclusion, i.e. whether a constructor is included in/covered by another,
+//! is straightforward and defined in [`Constructor::is_covered_by`].
+//!
+//! Constructor splitting is mentioned in [`super::usefulness`] but not detailed. We describe it
+//! precisely here.
+//!
+//!
+//!
+//! # Constructor grouping and splitting
+//!
+//! As explained in the corresponding section in [`super::usefulness`], to make usefulness tractable
+//! we need to group together constructors that have the same effect when they are used to
+//! specialize the matrix.
+//!
+//! Example:
+//! ```compile_fail,E0004
+//! match (0, false) {
+//!     (0 ..=100, true) => {}
+//!     (50..=150, false) => {}
+//!     (0 ..=200, _) => {}
+//! }
+//! ```
+//!
+//! In this example we can restrict specialization to 5 cases: `0..50`, `50..=100`, `101..=150`,
+//! `151..=200` and `200..`.
+//!
+//! In [`super::usefulness`], we had said that `specialize` only takes value-only constructors. We
+//! now relax this restriction: we allow `specialize` to take constructors like `0..50` as long as
+//! we're careful to only do that with constructors that make sense. For example, `specialize(0..50,
+//! (0..=100, true))` is sensible, but `specialize(50..=200, (0..=100, true))` is not.
+//!
+//! Constructor splitting looks at the constructors in the first column of the matrix and constructs
+//! such a sensible set of constructors. Formally, we want to find a smallest disjoint set of
+//! constructors:
+//! - Whose union covers the whole type, and
+//! - That have no non-trivial intersection with any of the constructors in the column (i.e. they're
+//!     each either disjoint with or covered by any given column constructor).
+//!
+//! We compute this in two steps: first [`ConstructorSet::for_ty`] determines the set of all
+//! possible constructors for the type. Then [`ConstructorSet::split`] looks at the column of
+//! constructors and splits the set into groups accordingly. The precise invariants of
+//! [`ConstructorSet::split`] is described in [`SplitConstructorSet`].
+//!
+//! Constructor splitting has two interesting special cases: integer range splitting (see
+//! [`IntRange::split`]) and slice splitting (see [`Slice::split`]).
+//!
+//!
+//!
+//! # The `Missing` constructor
+//!
+//! We detail a special case of constructor splitting that is a bit subtle. Take the following:
+//!
+//! ```
+//! enum Direction { North, South, East, West }
+//! # let wind = (Direction::North, 0u8);
+//! match wind {
+//!     (Direction::North, 50..) => {}
+//!     (_, _) => {}
+//! }
+//! ```
+//!
+//! Here we expect constructor splitting to output two cases: `North`, and "everything else". This
+//! "everything else" is represented by [`Constructor::Missing`]. Unlike other constructors, it's a
+//! bit contextual: to know the exact list of constructors it represents we have to look at the
+//! column. In practice however we don't need to, because by construction it only matches rows that
+//! have wildcards. This is how this constructor is special: the only constructor that covers it is
+//! `Wildcard`.
+//!
+//! The only place where we care about which constructors `Missing` represents is in diagnostics
+//! (see `super::usefulness::WitnessMatrix::apply_constructor`).
+//!
+//! We choose whether to specialize with `Missing` in
+//! `super::usefulness::compute_exhaustiveness_and_reachability`.
+//!
+//!
+//!
+//! ## Empty types, empty constructors, and the `exhaustive_patterns` feature
+//!
+//! An empty type is a type that has no valid value, like `!`, `enum Void {}`, or `Result<!, !>`.
+//! They require careful handling.
+//!
+//! First, for soundness reasons related to the possible existence of invalid values, by default we
+//! don't treat empty types as empty. We force them to be matched with wildcards. Except if the
+//! `exhaustive_patterns` feature is turned on, in which case we do treat them as empty. And also
+//! except if the type has no constructors (like `enum Void {}` but not like `Result<!, !>`), we
+//! specifically allow `match void {}` to be exhaustive. There are additionally considerations of
+//! place validity that are handled in `super::usefulness`. Yes this is a bit tricky.
+//!
+//! The second thing is that regardless of the above, it is always allowed to use all the
+//! constructors of a type. For example, all the following is ok:
+//!
+//! ```rust,ignore(example)
+//! # #![feature(never_type)]
+//! # #![feature(exhaustive_patterns)]
+//! fn foo(x: Option<!>) {
+//!   match x {
+//!     None => {}
+//!     Some(_) => {}
+//!   }
+//! }
+//! fn bar(x: &[!]) -> u32 {
+//!   match x {
+//!     [] => 1,
+//!     [_] => 2,
+//!     [_, _] => 3,
+//!   }
+//! }
+//! ```
+//!
+//! Moreover, take the following:
+//!
+//! ```rust
+//! # #![feature(never_type)]
+//! # #![feature(exhaustive_patterns)]
+//! # let x = None::<!>;
+//! match x {
+//!   None => {}
+//! }
+//! ```
+//!
+//! On a normal type, we would identify `Some` as missing and tell the user. If `x: Option<!>`
+//! however (and `exhaustive_patterns` is on), it's ok to omit `Some`. When listing the constructors
+//! of a type, we must therefore track which can be omitted.
+//!
+//! Let's call "empty" a constructor that matches no valid value for the type, like `Some` for the
+//! type `Option<!>`. What this all means is that `ConstructorSet` must know which constructors are
+//! empty. The difference between empty and nonempty constructors is that empty constructors need
+//! not be present for the match to be exhaustive.
+//!
+//! A final remark: empty constructors of arity 0 break specialization, we must avoid them. The
+//! reason is that if we specialize by them, nothing remains to witness the emptiness; the rest of
+//! the algorithm can't distinguish them from a nonempty constructor. The only known case where this
+//! could happen is the `[..]` pattern on `[!; N]` with `N > 0` so we must take care to not emit it.
+//!
+//! This is all handled by [`ConstructorSet::for_ty`] and [`ConstructorSet::split`]. The invariants
+//! of [`SplitConstructorSet`] are also of interest.
+//!
+//!
+//!
+//! ## Opaque patterns
+//!
+//! Some patterns, such as constants that are not allowed to be matched structurally, cannot be
+//! inspected, which we handle with `Constructor::Opaque`. Since we know nothing of these patterns,
+//! we assume they never cover each other. In order to respect the invariants of
+//! [`SplitConstructorSet`], we give each `Opaque` constructor a unique id so we can recognize it.
+
+use std::cmp::{self, max, min, Ordering};
+use std::fmt;
+use std::iter::once;
+
+use smallvec::SmallVec;
+
+use rustc_apfloat::ieee::{DoubleS, IeeeFloat, SingleS};
+use rustc_data_structures::fx::FxHashSet;
+use rustc_hir::RangeEnd;
+use rustc_index::IndexVec;
+use rustc_middle::middle::stability::EvalResult;
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::Scalar;
+use rustc_middle::thir::{Pat, PatKind, PatRange, PatRangeBoundary};
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::DUMMY_SP;
+use rustc_target::abi::{Integer, VariantIdx, FIRST_VARIANT};
+
+use self::Constructor::*;
+use self::MaybeInfiniteInt::*;
+use self::SliceKind::*;
+
+use crate::pat::Fields;
+use crate::usefulness::{MatchCheckCtxt, PatCtxt};
+
+/// Whether we have seen a constructor in the column or not.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+enum Presence {
+    Unseen,
+    Seen,
+}
+
+/// A possibly infinite integer. Values are encoded such that the ordering on `u128` matches the
+/// natural order on the original type. For example, `-128i8` is encoded as `0` and `127i8` as
+/// `255`. See `signed_bias` for details.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum MaybeInfiniteInt {
+    NegInfinity,
+    /// Encoded value. DO NOT CONSTRUCT BY HAND; use `new_finite`.
+    Finite(u128),
+    /// The integer after `u128::MAX`. We need it to represent `x..=u128::MAX` as an exclusive range.
+    JustAfterMax,
+    PosInfinity,
+}
+
+impl MaybeInfiniteInt {
+    // The return value of `signed_bias` should be XORed with a value to encode/decode it.
+    fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
+        match *ty.kind() {
+            ty::Int(ity) => {
+                let bits = Integer::from_int_ty(&tcx, ity).size().bits() as u128;
+                1u128 << (bits - 1)
+            }
+            _ => 0,
+        }
+    }
+
+    fn new_finite(tcx: TyCtxt<'_>, ty: Ty<'_>, bits: u128) -> Self {
+        let bias = Self::signed_bias(tcx, ty);
+        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
+        // type-independent.
+        let x = bits ^ bias;
+        Finite(x)
+    }
+    pub(crate) fn from_pat_range_bdy<'tcx>(
+        bdy: PatRangeBoundary<'tcx>,
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Self {
+        match bdy {
+            PatRangeBoundary::NegInfinity => NegInfinity,
+            PatRangeBoundary::Finite(value) => {
+                let bits = value.eval_bits(tcx, param_env);
+                Self::new_finite(tcx, ty, bits)
+            }
+            PatRangeBoundary::PosInfinity => PosInfinity,
+        }
+    }
+
+    /// Used only for diagnostics.
+    /// Note: it is possible to get `isize/usize::MAX+1` here, as explained in the doc for
+    /// [`IntRange::split`]. This cannot be represented as a `Const`, so we represent it with
+    /// `PosInfinity`.
+    fn to_diagnostic_pat_range_bdy<'tcx>(
+        self,
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+    ) -> PatRangeBoundary<'tcx> {
+        match self {
+            NegInfinity => PatRangeBoundary::NegInfinity,
+            Finite(x) => {
+                let bias = Self::signed_bias(tcx, ty);
+                let bits = x ^ bias;
+                let size = ty.primitive_size(tcx);
+                match Scalar::try_from_uint(bits, size) {
+                    Some(scalar) => {
+                        let value = mir::Const::from_scalar(tcx, scalar, ty);
+                        PatRangeBoundary::Finite(value)
+                    }
+                    // The value doesn't fit. Since `x >= 0` and 0 always encodes the minimum value
+                    // for a type, the problem isn't that the value is too small. So it must be too
+                    // large.
+                    None => PatRangeBoundary::PosInfinity,
+                }
+            }
+            JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
+        }
+    }
+
+    /// Note: this will not turn a finite value into an infinite one or vice-versa.
+    pub fn minus_one(self) -> Self {
+        match self {
+            Finite(n) => match n.checked_sub(1) {
+                Some(m) => Finite(m),
+                None => bug!(),
+            },
+            JustAfterMax => Finite(u128::MAX),
+            x => x,
+        }
+    }
+    /// Note: this will not turn a finite value into an infinite one or vice-versa.
+    pub fn plus_one(self) -> Self {
+        match self {
+            Finite(n) => match n.checked_add(1) {
+                Some(m) => Finite(m),
+                None => JustAfterMax,
+            },
+            JustAfterMax => bug!(),
+            x => x,
+        }
+    }
+}
+
+/// An exclusive interval, used for precise integer exhaustiveness checking. `IntRange`s always
+/// store a contiguous range.
+///
+/// `IntRange` is never used to encode an empty range or a "range" that wraps around the (offset)
+/// space: i.e., `range.lo < range.hi`.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub struct IntRange {
+    pub(crate) lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
+    pub(crate) hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
+}
+
+impl IntRange {
+    #[inline]
+    pub(super) fn is_integral(ty: Ty<'_>) -> bool {
+        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_))
+    }
+
+    /// Best effort; will not know that e.g. `255u8..` is a singleton.
+    pub fn is_singleton(&self) -> bool {
+        // Since `lo` and `hi` can't be the same `Infinity` and `plus_one` never changes from finite
+        // to infinite, this correctly only detects ranges that contain exacly one `Finite(x)`.
+        self.lo.plus_one() == self.hi
+    }
+
+    #[inline]
+    pub fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
+        let x = MaybeInfiniteInt::new_finite(tcx, ty, bits);
+        IntRange { lo: x, hi: x.plus_one() }
+    }
+
+    #[inline]
+    pub fn from_range(lo: MaybeInfiniteInt, mut hi: MaybeInfiniteInt, end: RangeEnd) -> IntRange {
+        if end == RangeEnd::Included {
+            hi = hi.plus_one();
+        }
+        if lo >= hi {
+            // This should have been caught earlier by E0030.
+            bug!("malformed range pattern: {lo:?}..{hi:?}");
+        }
+        IntRange { lo, hi }
+    }
+
+    fn is_subrange(&self, other: &Self) -> bool {
+        other.lo <= self.lo && self.hi <= other.hi
+    }
+
+    fn intersection(&self, other: &Self) -> Option<Self> {
+        if self.lo < other.hi && other.lo < self.hi {
+            Some(IntRange { lo: max(self.lo, other.lo), hi: min(self.hi, other.hi) })
+        } else {
+            None
+        }
+    }
+
+    /// Partition a range of integers into disjoint subranges. This does constructor splitting for
+    /// integer ranges as explained at the top of the file.
+    ///
+    /// This returns an output that covers `self`. The output is split so that the only
+    /// intersections between an output range and a column range are inclusions. No output range
+    /// straddles the boundary of one of the inputs.
+    ///
+    /// Additionally, we track for each output range whether it is covered by one of the column ranges or not.
+    ///
+    /// The following input:
+    /// ```text
+    ///   (--------------------------) // `self`
+    /// (------) (----------)    (-)
+    ///     (------) (--------)
+    /// ```
+    /// is first intersected with `self`:
+    /// ```text
+    ///   (--------------------------) // `self`
+    ///   (----) (----------)    (-)
+    ///     (------) (--------)
+    /// ```
+    /// and then iterated over as follows:
+    /// ```text
+    ///   (-(--)-(-)-(------)-)--(-)-
+    /// ```
+    /// where each sequence of dashes is an output range, and dashes outside parentheses are marked
+    /// as `Presence::Missing`.
+    ///
+    /// ## `isize`/`usize`
+    ///
+    /// Whereas a wildcard of type `i32` stands for the range `i32::MIN..=i32::MAX`, a `usize`
+    /// wildcard stands for `0..PosInfinity` and a `isize` wildcard stands for
+    /// `NegInfinity..PosInfinity`. In other words, as far as `IntRange` is concerned, there are
+    /// values before `isize::MIN` and after `usize::MAX`/`isize::MAX`.
+    /// This is to avoid e.g. `0..(u32::MAX as usize)` from being exhaustive on one architecture and
+    /// not others. This was decided in <https://github.com/rust-lang/rfcs/pull/2591>.
+    ///
+    /// These infinities affect splitting subtly: it is possible to get `NegInfinity..0` and
+    /// `usize::MAX+1..PosInfinity` in the output. Diagnostics must be careful to handle these
+    /// fictitious ranges sensibly.
+    fn split(
+        &self,
+        column_ranges: impl Iterator<Item = IntRange>,
+    ) -> impl Iterator<Item = (Presence, IntRange)> {
+        // The boundaries of ranges in `column_ranges` intersected with `self`.
+        // We do parenthesis matching for input ranges. A boundary counts as +1 if it starts
+        // a range and -1 if it ends it. When the count is > 0 between two boundaries, we
+        // are within an input range.
+        let mut boundaries: Vec<(MaybeInfiniteInt, isize)> = column_ranges
+            .filter_map(|r| self.intersection(&r))
+            .flat_map(|r| [(r.lo, 1), (r.hi, -1)])
+            .collect();
+        // We sort by boundary, and for each boundary we sort the "closing parentheses" first. The
+        // order of +1/-1 for a same boundary value is actually irrelevant, because we only look at
+        // the accumulated count between distinct boundary values.
+        boundaries.sort_unstable();
+
+        // Accumulate parenthesis counts.
+        let mut paren_counter = 0isize;
+        // Gather pairs of adjacent boundaries.
+        let mut prev_bdy = self.lo;
+        boundaries
+            .into_iter()
+            // End with the end of the range. The count is ignored.
+            .chain(once((self.hi, 0)))
+            // List pairs of adjacent boundaries and the count between them.
+            .map(move |(bdy, delta)| {
+                // `delta` affects the count as we cross `bdy`, so the relevant count between
+                // `prev_bdy` and `bdy` is untouched by `delta`.
+                let ret = (prev_bdy, paren_counter, bdy);
+                prev_bdy = bdy;
+                paren_counter += delta;
+                ret
+            })
+            // Skip empty ranges.
+            .filter(|&(prev_bdy, _, bdy)| prev_bdy != bdy)
+            // Convert back to ranges.
+            .map(move |(prev_bdy, paren_count, bdy)| {
+                use Presence::*;
+                let presence = if paren_count > 0 { Seen } else { Unseen };
+                let range = IntRange { lo: prev_bdy, hi: bdy };
+                (presence, range)
+            })
+    }
+
+    /// Whether the range denotes the fictitious values before `isize::MIN` or after
+    /// `usize::MAX`/`isize::MAX` (see doc of [`IntRange::split`] for why these exist).
+    pub fn is_beyond_boundaries<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool {
+        ty.is_ptr_sized_integral() && {
+            // The two invalid ranges are `NegInfinity..isize::MIN` (represented as
+            // `NegInfinity..0`), and `{u,i}size::MAX+1..PosInfinity`. `to_diagnostic_pat_range_bdy`
+            // converts `MAX+1` to `PosInfinity`, and we couldn't have `PosInfinity` in `self.lo`
+            // otherwise.
+            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
+            matches!(lo, PatRangeBoundary::PosInfinity)
+                || matches!(self.hi, MaybeInfiniteInt::Finite(0))
+        }
+    }
+    /// Only used for displaying the range.
+    pub(super) fn to_diagnostic_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
+        let kind = if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
+            PatKind::Wild
+        } else if self.is_singleton() {
+            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
+            let value = lo.as_finite().unwrap();
+            PatKind::Constant { value }
+        } else {
+            // We convert to an inclusive range for diagnostics.
+            let mut end = RangeEnd::Included;
+            let mut lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
+            if matches!(lo, PatRangeBoundary::PosInfinity) {
+                // The only reason to get `PosInfinity` here is the special case where
+                // `to_diagnostic_pat_range_bdy` found `{u,i}size::MAX+1`. So the range denotes the
+                // fictitious values after `{u,i}size::MAX` (see [`IntRange::split`] for why we do
+                // this). We show this to the user as `usize::MAX..` which is slightly incorrect but
+                // probably clear enough.
+                let c = ty.numeric_max_val(tcx).unwrap();
+                let value = mir::Const::from_ty_const(c, tcx);
+                lo = PatRangeBoundary::Finite(value);
+            }
+            let hi = if matches!(self.hi, MaybeInfiniteInt::Finite(0)) {
+                // The range encodes `..ty::MIN`, so we can't convert it to an inclusive range.
+                end = RangeEnd::Excluded;
+                self.hi
+            } else {
+                self.hi.minus_one()
+            };
+            let hi = hi.to_diagnostic_pat_range_bdy(ty, tcx);
+            PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
+        };
+
+        Pat { ty, span: DUMMY_SP, kind }
+    }
+}
+
+/// Note: this will render signed ranges incorrectly. To render properly, convert to a pattern
+/// first.
+impl fmt::Debug for IntRange {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Finite(lo) = self.lo {
+            write!(f, "{lo}")?;
+        }
+        write!(f, "{}", RangeEnd::Excluded)?;
+        if let Finite(hi) = self.hi {
+            write!(f, "{hi}")?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum SliceKind {
+    /// Patterns of length `n` (`[x, y]`).
+    FixedLen(usize),
+    /// Patterns using the `..` notation (`[x, .., y]`).
+    /// Captures any array constructor of `length >= i + j`.
+    /// In the case where `array_len` is `Some(_)`,
+    /// this indicates that we only care about the first `i` and the last `j` values of the array,
+    /// and everything in between is a wildcard `_`.
+    VarLen(usize, usize),
+}
+
+impl SliceKind {
+    fn arity(self) -> usize {
+        match self {
+            FixedLen(length) => length,
+            VarLen(prefix, suffix) => prefix + suffix,
+        }
+    }
+
+    /// Whether this pattern includes patterns of length `other_len`.
+    fn covers_length(self, other_len: usize) -> bool {
+        match self {
+            FixedLen(len) => len == other_len,
+            VarLen(prefix, suffix) => prefix + suffix <= other_len,
+        }
+    }
+}
+
+/// A constructor for array and slice patterns.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct Slice {
+    /// `None` if the matched value is a slice, `Some(n)` if it is an array of size `n`.
+    pub(crate) array_len: Option<usize>,
+    /// The kind of pattern it is: fixed-length `[x, y]` or variable length `[x, .., y]`.
+    pub(crate) kind: SliceKind,
+}
+
+impl Slice {
+    pub fn new(array_len: Option<usize>, kind: SliceKind) -> Self {
+        let kind = match (array_len, kind) {
+            // If the middle `..` has length 0, we effectively have a fixed-length pattern.
+            (Some(len), VarLen(prefix, suffix)) if prefix + suffix == len => FixedLen(len),
+            (Some(len), VarLen(prefix, suffix)) if prefix + suffix > len => bug!(
+                "Slice pattern of length {} longer than its array length {len}",
+                prefix + suffix
+            ),
+            _ => kind,
+        };
+        Slice { array_len, kind }
+    }
+
+    pub(crate) fn arity(self) -> usize {
+        self.kind.arity()
+    }
+
+    /// See `Constructor::is_covered_by`
+    fn is_covered_by(self, other: Self) -> bool {
+        other.kind.covers_length(self.arity())
+    }
+
+    /// This computes constructor splitting for variable-length slices, as explained at the top of
+    /// the file.
+    ///
+    /// A slice pattern `[x, .., y]` behaves like the infinite or-pattern `[x, y] | [x, _, y] | [x,
+    /// _, _, y] | etc`. The corresponding value constructors are fixed-length array constructors of
+    /// corresponding lengths. We obviously can't list this infinitude of constructors.
+    /// Thankfully, it turns out that for each finite set of slice patterns, all sufficiently large
+    /// array lengths are equivalent.
+    ///
+    /// Let's look at an example, where we are trying to split the last pattern:
+    /// ```
+    /// # fn foo(x: &[bool]) {
+    /// match x {
+    ///     [true, true, ..] => {}
+    ///     [.., false, false] => {}
+    ///     [..] => {}
+    /// }
+    /// # }
+    /// ```
+    /// Here are the results of specialization for the first few lengths:
+    /// ```
+    /// # fn foo(x: &[bool]) { match x {
+    /// // length 0
+    /// [] => {}
+    /// // length 1
+    /// [_] => {}
+    /// // length 2
+    /// [true, true] => {}
+    /// [false, false] => {}
+    /// [_, _] => {}
+    /// // length 3
+    /// [true, true,  _    ] => {}
+    /// [_,    false, false] => {}
+    /// [_,    _,     _    ] => {}
+    /// // length 4
+    /// [true, true, _,     _    ] => {}
+    /// [_,    _,    false, false] => {}
+    /// [_,    _,    _,     _    ] => {}
+    /// // length 5
+    /// [true, true, _, _,     _    ] => {}
+    /// [_,    _,    _, false, false] => {}
+    /// [_,    _,    _, _,     _    ] => {}
+    /// # _ => {}
+    /// # }}
+    /// ```
+    ///
+    /// We see that above length 4, we are simply inserting columns full of wildcards in the middle.
+    /// This means that specialization and witness computation with slices of length `l >= 4` will
+    /// give equivalent results regardless of `l`. This applies to any set of slice patterns: there
+    /// will be a length `L` above which all lengths behave the same. This is exactly what we need
+    /// for constructor splitting.
+    ///
+    /// A variable-length slice pattern covers all lengths from its arity up to infinity. As we just
+    /// saw, we can split this in two: lengths below `L` are treated individually with a
+    /// fixed-length slice each; lengths above `L` are grouped into a single variable-length slice
+    /// constructor.
+    ///
+    /// For each variable-length slice pattern `p` with a prefix of length `plₚ` and suffix of
+    /// length `slₚ`, only the first `plₚ` and the last `slₚ` elements are examined. Therefore, as
+    /// long as `L` is positive (to avoid concerns about empty types), all elements after the
+    /// maximum prefix length and before the maximum suffix length are not examined by any
+    /// variable-length pattern, and therefore can be ignored. This gives us a way to compute `L`.
+    ///
+    /// Additionally, if fixed-length patterns exist, we must pick an `L` large enough to miss them,
+    /// so we can pick `L = max(max(FIXED_LEN)+1, max(PREFIX_LEN) + max(SUFFIX_LEN))`.
+    /// `max_slice` below will be made to have this arity `L`.
+    ///
+    /// If `self` is fixed-length, it is returned as-is.
+    ///
+    /// Additionally, we track for each output slice whether it is covered by one of the column slices or not.
+    fn split(
+        self,
+        column_slices: impl Iterator<Item = Slice>,
+    ) -> impl Iterator<Item = (Presence, Slice)> {
+        // Range of lengths below `L`.
+        let smaller_lengths;
+        let arity = self.arity();
+        let mut max_slice = self.kind;
+        // Tracks the smallest variable-length slice we've seen. Any slice arity above it is
+        // therefore `Presence::Seen` in the column.
+        let mut min_var_len = usize::MAX;
+        // Tracks the fixed-length slices we've seen, to mark them as `Presence::Seen`.
+        let mut seen_fixed_lens = FxHashSet::default();
+        match &mut max_slice {
+            VarLen(max_prefix_len, max_suffix_len) => {
+                // A length larger than any fixed-length slice encountered.
+                // We start at 1 in case the subtype is empty because in that case the zero-length
+                // slice must be treated separately from the rest.
+                let mut fixed_len_upper_bound = 1;
+                // We grow `max_slice` to be larger than all slices encountered, as described above.
+                // `L` is `max_slice.arity()`. For diagnostics, we keep the prefix and suffix
+                // lengths separate.
+                for slice in column_slices {
+                    match slice.kind {
+                        FixedLen(len) => {
+                            fixed_len_upper_bound = cmp::max(fixed_len_upper_bound, len + 1);
+                            seen_fixed_lens.insert(len);
+                        }
+                        VarLen(prefix, suffix) => {
+                            *max_prefix_len = cmp::max(*max_prefix_len, prefix);
+                            *max_suffix_len = cmp::max(*max_suffix_len, suffix);
+                            min_var_len = cmp::min(min_var_len, prefix + suffix);
+                        }
+                    }
+                }
+                // If `fixed_len_upper_bound >= L`, we set `L` to `fixed_len_upper_bound`.
+                if let Some(delta) =
+                    fixed_len_upper_bound.checked_sub(*max_prefix_len + *max_suffix_len)
+                {
+                    *max_prefix_len += delta
+                }
+
+                // We cap the arity of `max_slice` at the array size.
+                match self.array_len {
+                    Some(len) if max_slice.arity() >= len => max_slice = FixedLen(len),
+                    _ => {}
+                }
+
+                smaller_lengths = match self.array_len {
+                    // The only admissible fixed-length slice is one of the array size. Whether `max_slice`
+                    // is fixed-length or variable-length, it will be the only relevant slice to output
+                    // here.
+                    Some(_) => 0..0, // empty range
+                    // We need to cover all arities in the range `(arity..infinity)`. We split that
+                    // range into two: lengths smaller than `max_slice.arity()` are treated
+                    // independently as fixed-lengths slices, and lengths above are captured by
+                    // `max_slice`.
+                    None => self.arity()..max_slice.arity(),
+                };
+            }
+            FixedLen(_) => {
+                // No need to split here. We only track presence.
+                for slice in column_slices {
+                    match slice.kind {
+                        FixedLen(len) => {
+                            if len == arity {
+                                seen_fixed_lens.insert(len);
+                            }
+                        }
+                        VarLen(prefix, suffix) => {
+                            min_var_len = cmp::min(min_var_len, prefix + suffix);
+                        }
+                    }
+                }
+                smaller_lengths = 0..0;
+            }
+        };
+
+        smaller_lengths.map(FixedLen).chain(once(max_slice)).map(move |kind| {
+            let arity = kind.arity();
+            let seen = if min_var_len <= arity || seen_fixed_lens.contains(&arity) {
+                Presence::Seen
+            } else {
+                Presence::Unseen
+            };
+            (seen, Slice::new(self.array_len, kind))
+        })
+    }
+}
+
+/// A globally unique id to distinguish `Opaque` patterns.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct OpaqueId(u32);
+
+impl OpaqueId {
+    pub fn new() -> Self {
+        use std::sync::atomic::{AtomicU32, Ordering};
+        static OPAQUE_ID: AtomicU32 = AtomicU32::new(0);
+        OpaqueId(OPAQUE_ID.fetch_add(1, Ordering::SeqCst))
+    }
+}
+
+/// A value can be decomposed into a constructor applied to some fields. This struct represents
+/// the constructor. See also `Fields`.
+///
+/// `pat_constructor` retrieves the constructor corresponding to a pattern.
+/// `specialize_constructor` returns the list of fields corresponding to a pattern, given a
+/// constructor. `Constructor::apply` reconstructs the pattern from a pair of `Constructor` and
+/// `Fields`.
+#[derive(Clone, Debug, PartialEq)]
+pub enum Constructor<'tcx> {
+    /// The constructor for patterns that have a single constructor, like tuples, struct patterns,
+    /// and references. Fixed-length arrays are treated separately with `Slice`.
+    Single,
+    /// Enum variants.
+    Variant(VariantIdx),
+    /// Booleans
+    Bool(bool),
+    /// Ranges of integer literal values (`2`, `2..=5` or `2..5`).
+    IntRange(IntRange),
+    /// Ranges of floating-point literal values (`2.0..=5.2`).
+    F32Range(IeeeFloat<SingleS>, IeeeFloat<SingleS>, RangeEnd),
+    F64Range(IeeeFloat<DoubleS>, IeeeFloat<DoubleS>, RangeEnd),
+    /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately.
+    Str(mir::Const<'tcx>),
+    /// Array and slice patterns.
+    Slice(Slice),
+    /// Constants that must not be matched structurally. They are treated as black boxes for the
+    /// purposes of exhaustiveness: we must not inspect them, and they don't count towards making a
+    /// match exhaustive.
+    /// Carries an id that must be unique within a match. We need this to ensure the invariants of
+    /// [`SplitConstructorSet`].
+    Opaque(OpaqueId),
+    /// Or-pattern.
+    Or,
+    /// Wildcard pattern.
+    Wildcard,
+    /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. Also used
+    /// for those types for which we cannot list constructors explicitly, like `f64` and `str`.
+    NonExhaustive,
+    /// Fake extra constructor for variants that should not be mentioned in diagnostics.
+    /// We use this for variants behind an unstable gate as well as
+    /// `#[doc(hidden)]` ones.
+    Hidden,
+    /// Fake extra constructor for constructors that are not seen in the matrix, as explained at the
+    /// top of the file.
+    Missing,
+}
+
+impl<'tcx> Constructor<'tcx> {
+    pub(super) fn is_non_exhaustive(&self) -> bool {
+        matches!(self, NonExhaustive)
+    }
+
+    pub(super) fn as_variant(&self) -> Option<VariantIdx> {
+        match self {
+            Variant(i) => Some(*i),
+            _ => None,
+        }
+    }
+    fn as_bool(&self) -> Option<bool> {
+        match self {
+            Bool(b) => Some(*b),
+            _ => None,
+        }
+    }
+    pub(super) fn as_int_range(&self) -> Option<&IntRange> {
+        match self {
+            IntRange(range) => Some(range),
+            _ => None,
+        }
+    }
+    fn as_slice(&self) -> Option<Slice> {
+        match self {
+            Slice(slice) => Some(*slice),
+            _ => None,
+        }
+    }
+
+    pub(crate) fn variant_index_for_adt(&self, adt: ty::AdtDef<'tcx>) -> VariantIdx {
+        match *self {
+            Variant(idx) => idx,
+            Single => {
+                assert!(!adt.is_enum());
+                FIRST_VARIANT
+            }
+            _ => bug!("bad constructor {:?} for adt {:?}", self, adt),
+        }
+    }
+
+    /// The number of fields for this constructor. This must be kept in sync with
+    /// `Fields::wildcards`.
+    pub(crate) fn arity(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> usize {
+        match self {
+            Single | Variant(_) => match pcx.ty.kind() {
+                ty::Tuple(fs) => fs.len(),
+                ty::Ref(..) => 1,
+                ty::Adt(adt, ..) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        1
+                    } else {
+                        let variant = &adt.variant(self.variant_index_for_adt(*adt));
+                        Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant).count()
+                    }
+                }
+                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx.ty),
+            },
+            Slice(slice) => slice.arity(),
+            Bool(..)
+            | IntRange(..)
+            | F32Range(..)
+            | F64Range(..)
+            | Str(..)
+            | Opaque(..)
+            | NonExhaustive
+            | Hidden
+            | Missing { .. }
+            | Wildcard => 0,
+            Or => bug!("The `Or` constructor doesn't have a fixed arity"),
+        }
+    }
+
+    /// Returns whether `self` is covered by `other`, i.e. whether `self` is a subset of `other`.
+    /// For the simple cases, this is simply checking for equality. For the "grouped" constructors,
+    /// this checks for inclusion.
+    // We inline because this has a single call site in `Matrix::specialize_constructor`.
+    #[inline]
+    pub(crate) fn is_covered_by<'p>(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, other: &Self) -> bool {
+        match (self, other) {
+            (Wildcard, _) => {
+                span_bug!(
+                    pcx.cx.scrut_span,
+                    "Constructor splitting should not have returned `Wildcard`"
+                )
+            }
+            // Wildcards cover anything
+            (_, Wildcard) => true,
+            // Only a wildcard pattern can match these special constructors.
+            (Missing { .. } | NonExhaustive | Hidden, _) => false,
+
+            (Single, Single) => true,
+            (Variant(self_id), Variant(other_id)) => self_id == other_id,
+            (Bool(self_b), Bool(other_b)) => self_b == other_b,
+
+            (IntRange(self_range), IntRange(other_range)) => self_range.is_subrange(other_range),
+            (F32Range(self_from, self_to, self_end), F32Range(other_from, other_to, other_end)) => {
+                self_from.ge(other_from)
+                    && match self_to.partial_cmp(other_to) {
+                        Some(Ordering::Less) => true,
+                        Some(Ordering::Equal) => other_end == self_end,
+                        _ => false,
+                    }
+            }
+            (F64Range(self_from, self_to, self_end), F64Range(other_from, other_to, other_end)) => {
+                self_from.ge(other_from)
+                    && match self_to.partial_cmp(other_to) {
+                        Some(Ordering::Less) => true,
+                        Some(Ordering::Equal) => other_end == self_end,
+                        _ => false,
+                    }
+            }
+            (Str(self_val), Str(other_val)) => {
+                // FIXME Once valtrees are available we can directly use the bytes
+                // in the `Str` variant of the valtree for the comparison here.
+                self_val == other_val
+            }
+            (Slice(self_slice), Slice(other_slice)) => self_slice.is_covered_by(*other_slice),
+
+            // Opaque constructors don't interact with anything unless they come from the
+            // syntactically identical pattern.
+            (Opaque(self_id), Opaque(other_id)) => self_id == other_id,
+            (Opaque(..), _) | (_, Opaque(..)) => false,
+
+            _ => span_bug!(
+                pcx.cx.scrut_span,
+                "trying to compare incompatible constructors {:?} and {:?}",
+                self,
+                other
+            ),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum VariantVisibility {
+    /// Variant that doesn't fit the other cases, i.e. most variants.
+    Visible,
+    /// Variant behind an unstable gate or with the `#[doc(hidden)]` attribute. It will not be
+    /// mentioned in diagnostics unless the user mentioned it first.
+    Hidden,
+    /// Variant that matches no value. E.g. `Some::<Option<!>>` if the `exhaustive_patterns` feature
+    /// is enabled. Like `Hidden`, it will not be mentioned in diagnostics unless the user mentioned
+    /// it first.
+    Empty,
+}
+
+/// Describes the set of all constructors for a type. For details, in particular about the emptiness
+/// of constructors, see the top of the file.
+///
+/// In terms of division of responsibility, [`ConstructorSet::split`] handles all of the
+/// `exhaustive_patterns` feature.
+#[derive(Debug)]
+pub enum ConstructorSet {
+    /// The type has a single constructor, e.g. `&T` or a struct. `empty` tracks whether the
+    /// constructor is empty.
+    Single { empty: bool },
+    /// This type has the following list of constructors. If `variants` is empty and
+    /// `non_exhaustive` is false, don't use this; use `NoConstructors` instead.
+    Variants { variants: IndexVec<VariantIdx, VariantVisibility>, non_exhaustive: bool },
+    /// Booleans.
+    Bool,
+    /// The type is spanned by integer values. The range or ranges give the set of allowed values.
+    /// The second range is only useful for `char`.
+    Integers { range_1: IntRange, range_2: Option<IntRange> },
+    /// The type is matched by slices. `array_len` is the compile-time length of the array, if
+    /// known. If `subtype_is_empty`, all constructors are empty except possibly the zero-length
+    /// slice `[]`.
+    Slice { array_len: Option<usize>, subtype_is_empty: bool },
+    /// The constructors cannot be listed, and the type cannot be matched exhaustively. E.g. `str`,
+    /// floats.
+    Unlistable,
+    /// The type has no constructors (not even empty ones). This is `!` and empty enums.
+    NoConstructors,
+}
+
+/// Describes the result of analyzing the constructors in a column of a match.
+///
+/// `present` is morally the set of constructors present in the column, and `missing` is the set of
+/// constructors that exist in the type but are not present in the column.
+///
+/// More formally, if we discard wildcards from the column, this respects the following constraints:
+/// 1. the union of `present`, `missing` and `missing_empty` covers all the constructors of the type
+/// 2. each constructor in `present` is covered by something in the column
+/// 3. no constructor in `missing` or `missing_empty` is covered by anything in the column
+/// 4. each constructor in the column is equal to the union of one or more constructors in `present`
+/// 5. `missing` does not contain empty constructors (see discussion about emptiness at the top of
+///    the file);
+/// 6. `missing_empty` contains only empty constructors
+/// 7. constructors in `present`, `missing` and `missing_empty` are split for the column; in other
+///    words, they are either fully included in or fully disjoint from each constructor in the
+///    column. In yet other words, there are no non-trivial intersections like between `0..10` and
+///    `5..15`.
+///
+/// We must be particularly careful with weird constructors like `Opaque`: they're not formally part
+/// of the `ConstructorSet` for the type, yet if we forgot to include them in `present` we would be
+/// ignoring any row with `Opaque`s in the algorithm. Hence the importance of point 4.
+#[derive(Debug)]
+pub(super) struct SplitConstructorSet<'tcx> {
+    pub(super) present: SmallVec<[Constructor<'tcx>; 1]>,
+    pub(super) missing: Vec<Constructor<'tcx>>,
+    pub(super) missing_empty: Vec<Constructor<'tcx>>,
+}
+
+impl ConstructorSet {
+    /// Creates a set that represents all the constructors of `ty`.
+    ///
+    /// See at the top of the file for considerations of emptiness.
+    #[instrument(level = "debug", skip(cx), ret)]
+    pub fn for_ty<'p, 'tcx>(cx: &MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
+        let make_range = |start, end| {
+            IntRange::from_range(
+                MaybeInfiniteInt::new_finite(cx.tcx, ty, start),
+                MaybeInfiniteInt::new_finite(cx.tcx, ty, end),
+                RangeEnd::Included,
+            )
+        };
+        // This determines the set of all possible constructors for the type `ty`. For numbers,
+        // arrays and slices we use ranges and variable-length slices when appropriate.
+        match ty.kind() {
+            ty::Bool => Self::Bool,
+            ty::Char => {
+                // The valid Unicode Scalar Value ranges.
+                Self::Integers {
+                    range_1: make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
+                    range_2: Some(make_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
+                }
+            }
+            &ty::Int(ity) => {
+                let range = if ty.is_ptr_sized_integral() {
+                    // The min/max values of `isize` are not allowed to be observed.
+                    IntRange { lo: NegInfinity, hi: PosInfinity }
+                } else {
+                    let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
+                    let min = 1u128 << (bits - 1);
+                    let max = min - 1;
+                    make_range(min, max)
+                };
+                Self::Integers { range_1: range, range_2: None }
+            }
+            &ty::Uint(uty) => {
+                let range = if ty.is_ptr_sized_integral() {
+                    // The max value of `usize` is not allowed to be observed.
+                    let lo = MaybeInfiniteInt::new_finite(cx.tcx, ty, 0);
+                    IntRange { lo, hi: PosInfinity }
+                } else {
+                    let size = Integer::from_uint_ty(&cx.tcx, uty).size();
+                    let max = size.truncate(u128::MAX);
+                    make_range(0, max)
+                };
+                Self::Integers { range_1: range, range_2: None }
+            }
+            ty::Slice(sub_ty) => {
+                Self::Slice { array_len: None, subtype_is_empty: cx.is_uninhabited(*sub_ty) }
+            }
+            ty::Array(sub_ty, len) => {
+                // We treat arrays of a constant but unknown length like slices.
+                Self::Slice {
+                    array_len: len.try_eval_target_usize(cx.tcx, cx.param_env).map(|l| l as usize),
+                    subtype_is_empty: cx.is_uninhabited(*sub_ty),
+                }
+            }
+            ty::Adt(def, args) if def.is_enum() => {
+                let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(ty);
+                if def.variants().is_empty() && !is_declared_nonexhaustive {
+                    Self::NoConstructors
+                } else {
+                    let mut variants =
+                        IndexVec::from_elem(VariantVisibility::Visible, def.variants());
+                    for (idx, v) in def.variants().iter_enumerated() {
+                        let variant_def_id = def.variant(idx).def_id;
+                        // Visibly uninhabited variants.
+                        let is_inhabited = v
+                            .inhabited_predicate(cx.tcx, *def)
+                            .instantiate(cx.tcx, args)
+                            .apply(cx.tcx, cx.param_env, cx.module);
+                        // Variants that depend on a disabled unstable feature.
+                        let is_unstable = matches!(
+                            cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
+                            EvalResult::Deny { .. }
+                        );
+                        // Foreign `#[doc(hidden)]` variants.
+                        let is_doc_hidden =
+                            cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
+                        let visibility = if !is_inhabited {
+                            // FIXME: handle empty+hidden
+                            VariantVisibility::Empty
+                        } else if is_unstable || is_doc_hidden {
+                            VariantVisibility::Hidden
+                        } else {
+                            VariantVisibility::Visible
+                        };
+                        variants[idx] = visibility;
+                    }
+
+                    Self::Variants { variants, non_exhaustive: is_declared_nonexhaustive }
+                }
+            }
+            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => {
+                Self::Single { empty: cx.is_uninhabited(ty) }
+            }
+            ty::Never => Self::NoConstructors,
+            // This type is one for which we cannot list constructors, like `str` or `f64`.
+            // FIXME(Nadrieril): which of these are actually allowed?
+            ty::Float(_)
+            | ty::Str
+            | ty::Foreign(_)
+            | ty::RawPtr(_)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Closure(_, _)
+            | ty::Coroutine(_, _, _)
+            | ty::Alias(_, _)
+            | ty::Param(_)
+            | ty::Error(_) => Self::Unlistable,
+            ty::CoroutineWitness(_, _) | ty::Bound(_, _) | ty::Placeholder(_) | ty::Infer(_) => {
+                bug!("Encountered unexpected type in `ConstructorSet::for_ty`: {ty:?}")
+            }
+        }
+    }
+
+    /// This analyzes a column of constructors to 1/ determine which constructors of the type (if
+    /// any) are missing; 2/ split constructors to handle non-trivial intersections e.g. on ranges
+    /// or slices. This can get subtle; see [`SplitConstructorSet`] for details of this operation
+    /// and its invariants.
+    #[instrument(level = "debug", skip(self, pcx, ctors), ret)]
+    pub(super) fn split<'a, 'tcx>(
+        &self,
+        pcx: &PatCtxt<'_, '_, 'tcx>,
+        ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
+    ) -> SplitConstructorSet<'tcx>
+    where
+        'tcx: 'a,
+    {
+        let mut present: SmallVec<[_; 1]> = SmallVec::new();
+        // Empty constructors found missing.
+        let mut missing_empty = Vec::new();
+        // Nonempty constructors found missing.
+        let mut missing = Vec::new();
+        // Constructors in `ctors`, except wildcards and opaques.
+        let mut seen = Vec::new();
+        for ctor in ctors.cloned() {
+            match ctor {
+                Opaque(..) => present.push(ctor),
+                Wildcard => {} // discard wildcards
+                _ => seen.push(ctor),
+            }
+        }
+
+        match self {
+            ConstructorSet::Single { empty } => {
+                if !seen.is_empty() {
+                    present.push(Single);
+                } else if *empty {
+                    missing_empty.push(Single);
+                } else {
+                    missing.push(Single);
+                }
+            }
+            ConstructorSet::Variants { variants, non_exhaustive } => {
+                let seen_set: FxHashSet<_> = seen.iter().map(|c| c.as_variant().unwrap()).collect();
+                let mut skipped_a_hidden_variant = false;
+
+                for (idx, visibility) in variants.iter_enumerated() {
+                    let ctor = Variant(idx);
+                    if seen_set.contains(&idx) {
+                        present.push(ctor);
+                    } else {
+                        // We only put visible variants directly into `missing`.
+                        match visibility {
+                            VariantVisibility::Visible => missing.push(ctor),
+                            VariantVisibility::Hidden => skipped_a_hidden_variant = true,
+                            VariantVisibility::Empty => missing_empty.push(ctor),
+                        }
+                    }
+                }
+
+                if skipped_a_hidden_variant {
+                    missing.push(Hidden);
+                }
+                if *non_exhaustive {
+                    missing.push(NonExhaustive);
+                }
+            }
+            ConstructorSet::Bool => {
+                let mut seen_false = false;
+                let mut seen_true = false;
+                for b in seen.iter().map(|ctor| ctor.as_bool().unwrap()) {
+                    if b {
+                        seen_true = true;
+                    } else {
+                        seen_false = true;
+                    }
+                }
+                if seen_false {
+                    present.push(Bool(false));
+                } else {
+                    missing.push(Bool(false));
+                }
+                if seen_true {
+                    present.push(Bool(true));
+                } else {
+                    missing.push(Bool(true));
+                }
+            }
+            ConstructorSet::Integers { range_1, range_2 } => {
+                let seen_ranges: Vec<_> =
+                    seen.iter().map(|ctor| ctor.as_int_range().unwrap().clone()).collect();
+                for (seen, splitted_range) in range_1.split(seen_ranges.iter().cloned()) {
+                    match seen {
+                        Presence::Unseen => missing.push(IntRange(splitted_range)),
+                        Presence::Seen => present.push(IntRange(splitted_range)),
+                    }
+                }
+                if let Some(range_2) = range_2 {
+                    for (seen, splitted_range) in range_2.split(seen_ranges.into_iter()) {
+                        match seen {
+                            Presence::Unseen => missing.push(IntRange(splitted_range)),
+                            Presence::Seen => present.push(IntRange(splitted_range)),
+                        }
+                    }
+                }
+            }
+            ConstructorSet::Slice { array_len, subtype_is_empty } => {
+                let seen_slices = seen.iter().map(|c| c.as_slice().unwrap());
+                let base_slice = Slice::new(*array_len, VarLen(0, 0));
+                for (seen, splitted_slice) in base_slice.split(seen_slices) {
+                    let ctor = Slice(splitted_slice);
+                    match seen {
+                        Presence::Seen => present.push(ctor),
+                        Presence::Unseen => {
+                            if *subtype_is_empty && splitted_slice.arity() != 0 {
+                                // We have subpatterns of an empty type, so the constructor is
+                                // empty.
+                                missing_empty.push(ctor);
+                            } else {
+                                missing.push(ctor);
+                            }
+                        }
+                    }
+                }
+            }
+            ConstructorSet::Unlistable => {
+                // Since we can't list constructors, we take the ones in the column. This might list
+                // some constructors several times but there's not much we can do.
+                present.extend(seen);
+                missing.push(NonExhaustive);
+            }
+            ConstructorSet::NoConstructors => {
+                // In a `MaybeInvalid` place even an empty pattern may be reachable. We therefore
+                // add a dummy empty constructor here, which will be ignored if the place is
+                // `ValidOnly`.
+                missing_empty.push(NonExhaustive);
+            }
+        }
+
+        // We have now grouped all the constructors into 3 buckets: present, missing, missing_empty.
+        // In the absence of the `exhaustive_patterns` feature however, we don't count nested empty
+        // types as empty. Only non-nested `!` or `enum Foo {}` are considered empty.
+        if !pcx.cx.tcx.features().exhaustive_patterns
+            && !(pcx.is_top_level && matches!(self, Self::NoConstructors))
+        {
+            // Treat all missing constructors as nonempty.
+            missing.extend(missing_empty.drain(..));
+        }
+
+        SplitConstructorSet { present, missing, missing_empty }
+    }
+}
diff --git a/compiler/rustc_pattern_analysis/src/errors.rs b/compiler/rustc_pattern_analysis/src/errors.rs
new file mode 100644
index 00000000000..0dddcb505e8
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/errors.rs
@@ -0,0 +1,95 @@
+use crate::{pat::WitnessPat, usefulness::MatchCheckCtxt};
+
+use rustc_errors::{AddToDiagnostic, Diagnostic, SubdiagnosticMessage};
+use rustc_macros::{LintDiagnostic, Subdiagnostic};
+use rustc_middle::thir::Pat;
+use rustc_middle::ty::Ty;
+use rustc_span::Span;
+
+#[derive(Subdiagnostic)]
+#[label(pattern_analysis_uncovered)]
+pub struct Uncovered<'tcx> {
+    #[primary_span]
+    span: Span,
+    count: usize,
+    witness_1: Pat<'tcx>,
+    witness_2: Pat<'tcx>,
+    witness_3: Pat<'tcx>,
+    remainder: usize,
+}
+
+impl<'tcx> Uncovered<'tcx> {
+    pub fn new<'p>(
+        span: Span,
+        cx: &MatchCheckCtxt<'p, 'tcx>,
+        witnesses: Vec<WitnessPat<'tcx>>,
+    ) -> Self {
+        let witness_1 = witnesses.get(0).unwrap().to_diagnostic_pat(cx);
+        Self {
+            span,
+            count: witnesses.len(),
+            // Substitute dummy values if witnesses is smaller than 3. These will never be read.
+            witness_2: witnesses
+                .get(1)
+                .map(|w| w.to_diagnostic_pat(cx))
+                .unwrap_or_else(|| witness_1.clone()),
+            witness_3: witnesses
+                .get(2)
+                .map(|w| w.to_diagnostic_pat(cx))
+                .unwrap_or_else(|| witness_1.clone()),
+            witness_1,
+            remainder: witnesses.len().saturating_sub(3),
+        }
+    }
+}
+
+#[derive(LintDiagnostic)]
+#[diag(pattern_analysis_overlapping_range_endpoints)]
+#[note]
+pub struct OverlappingRangeEndpoints<'tcx> {
+    #[label]
+    pub range: Span,
+    #[subdiagnostic]
+    pub overlap: Vec<Overlap<'tcx>>,
+}
+
+pub struct Overlap<'tcx> {
+    pub span: Span,
+    pub range: Pat<'tcx>,
+}
+
+impl<'tcx> AddToDiagnostic for Overlap<'tcx> {
+    fn add_to_diagnostic_with<F>(self, diag: &mut Diagnostic, _: F)
+    where
+        F: Fn(&mut Diagnostic, SubdiagnosticMessage) -> SubdiagnosticMessage,
+    {
+        let Overlap { span, range } = self;
+
+        // FIXME(mejrs) unfortunately `#[derive(LintDiagnostic)]`
+        // does not support `#[subdiagnostic(eager)]`...
+        let message = format!("this range overlaps on `{range}`...");
+        diag.span_label(span, message);
+    }
+}
+
+#[derive(LintDiagnostic)]
+#[diag(pattern_analysis_non_exhaustive_omitted_pattern)]
+#[help]
+#[note]
+pub(crate) struct NonExhaustiveOmittedPattern<'tcx> {
+    pub scrut_ty: Ty<'tcx>,
+    #[subdiagnostic]
+    pub uncovered: Uncovered<'tcx>,
+}
+
+#[derive(LintDiagnostic)]
+#[diag(pattern_analysis_non_exhaustive_omitted_pattern_lint_on_arm)]
+#[help]
+pub(crate) struct NonExhaustiveOmittedPatternLintOnArm {
+    #[label]
+    pub lint_span: Span,
+    #[suggestion(code = "#[{lint_level}({lint_name})]\n", applicability = "maybe-incorrect")]
+    pub suggest_lint_on_match: Option<Span>,
+    pub lint_level: &'static str,
+    pub lint_name: &'static str,
+}
diff --git a/compiler/rustc_pattern_analysis/src/lib.rs b/compiler/rustc_pattern_analysis/src/lib.rs
new file mode 100644
index 00000000000..bf131a37809
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/lib.rs
@@ -0,0 +1,13 @@
+//! Analysis of patterns, notably match exhaustiveness checking.
+
+pub mod constructor;
+pub mod errors;
+pub mod pat;
+pub mod usefulness;
+
+#[macro_use]
+extern crate tracing;
+#[macro_use]
+extern crate rustc_middle;
+
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
new file mode 100644
index 00000000000..ded992cda93
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -0,0 +1,744 @@
+//! As explained in [`crate::usefulness`], values and patterns are made from constructors applied to
+//! fields. This file defines types that represent patterns in this way.
+use std::cell::Cell;
+use std::fmt;
+use std::iter::once;
+
+use smallvec::{smallvec, SmallVec};
+
+use rustc_data_structures::captures::Captures;
+use rustc_hir::RangeEnd;
+use rustc_index::Idx;
+use rustc_middle::mir;
+use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange};
+use rustc_middle::ty::{self, Ty, VariantDef};
+use rustc_span::{Span, DUMMY_SP};
+use rustc_target::abi::FieldIdx;
+
+use self::Constructor::*;
+use self::SliceKind::*;
+
+use crate::constructor::{Constructor, IntRange, MaybeInfiniteInt, OpaqueId, Slice, SliceKind};
+use crate::usefulness::{MatchCheckCtxt, PatCtxt};
+
+/// A value can be decomposed into a constructor applied to some fields. This struct represents
+/// those fields, generalized to allow patterns in each field. See also `Constructor`.
+///
+/// This is constructed for a constructor using [`Fields::wildcards()`]. The idea is that
+/// [`Fields::wildcards()`] constructs a list of fields where all entries are wildcards, and then
+/// given a pattern we fill some of the fields with its subpatterns.
+/// In the following example `Fields::wildcards` returns `[_, _, _, _]`. Then in
+/// `extract_pattern_arguments` we fill some of the entries, and the result is
+/// `[Some(0), _, _, _]`.
+/// ```compile_fail,E0004
+/// # fn foo() -> [Option<u8>; 4] { [None; 4] }
+/// let x: [Option<u8>; 4] = foo();
+/// match x {
+///     [Some(0), ..] => {}
+/// }
+/// ```
+///
+/// Note that the number of fields of a constructor may not match the fields declared in the
+/// original struct/variant. This happens if a private or `non_exhaustive` field is uninhabited,
+/// because the code mustn't observe that it is uninhabited. In that case that field is not
+/// included in `fields`. For that reason, when you have a `FieldIdx` you must use
+/// `index_with_declared_idx`.
+#[derive(Debug, Clone, Copy)]
+pub struct Fields<'p, 'tcx> {
+    fields: &'p [DeconstructedPat<'p, 'tcx>],
+}
+
+impl<'p, 'tcx> Fields<'p, 'tcx> {
+    fn empty() -> Self {
+        Fields { fields: &[] }
+    }
+
+    fn singleton(cx: &MatchCheckCtxt<'p, 'tcx>, field: DeconstructedPat<'p, 'tcx>) -> Self {
+        let field: &_ = cx.pattern_arena.alloc(field);
+        Fields { fields: std::slice::from_ref(field) }
+    }
+
+    pub fn from_iter(
+        cx: &MatchCheckCtxt<'p, 'tcx>,
+        fields: impl IntoIterator<Item = DeconstructedPat<'p, 'tcx>>,
+    ) -> Self {
+        let fields: &[_] = cx.pattern_arena.alloc_from_iter(fields);
+        Fields { fields }
+    }
+
+    fn wildcards_from_tys(
+        cx: &MatchCheckCtxt<'p, 'tcx>,
+        tys: impl IntoIterator<Item = Ty<'tcx>>,
+    ) -> Self {
+        Fields::from_iter(cx, tys.into_iter().map(|ty| DeconstructedPat::wildcard(ty, DUMMY_SP)))
+    }
+
+    // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
+    // uninhabited fields in order not to reveal the uninhabitedness of the whole variant.
+    // This lists the fields we keep along with their types.
+    pub(crate) fn list_variant_nonhidden_fields<'a>(
+        cx: &'a MatchCheckCtxt<'p, 'tcx>,
+        ty: Ty<'tcx>,
+        variant: &'a VariantDef,
+    ) -> impl Iterator<Item = (FieldIdx, Ty<'tcx>)> + Captures<'a> + Captures<'p> {
+        let ty::Adt(adt, args) = ty.kind() else { bug!() };
+        // Whether we must not match the fields of this variant exhaustively.
+        let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
+
+        variant.fields.iter().enumerate().filter_map(move |(i, field)| {
+            let ty = field.ty(cx.tcx, args);
+            // `field.ty()` doesn't normalize after substituting.
+            let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
+            let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
+            let is_uninhabited = cx.tcx.features().exhaustive_patterns && cx.is_uninhabited(ty);
+
+            if is_uninhabited && (!is_visible || is_non_exhaustive) {
+                None
+            } else {
+                Some((FieldIdx::new(i), ty))
+            }
+        })
+    }
+
+    /// Creates a new list of wildcard fields for a given constructor. The result must have a
+    /// length of `constructor.arity()`.
+    #[instrument(level = "trace")]
+    pub(super) fn wildcards(pcx: &PatCtxt<'_, 'p, 'tcx>, constructor: &Constructor<'tcx>) -> Self {
+        let ret = match constructor {
+            Single | Variant(_) => match pcx.ty.kind() {
+                ty::Tuple(fs) => Fields::wildcards_from_tys(pcx.cx, fs.iter()),
+                ty::Ref(_, rty, _) => Fields::wildcards_from_tys(pcx.cx, once(*rty)),
+                ty::Adt(adt, args) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        Fields::wildcards_from_tys(pcx.cx, once(args.type_at(0)))
+                    } else {
+                        let variant = &adt.variant(constructor.variant_index_for_adt(*adt));
+                        let tys = Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant)
+                            .map(|(_, ty)| ty);
+                        Fields::wildcards_from_tys(pcx.cx, tys)
+                    }
+                }
+                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx),
+            },
+            Slice(slice) => match *pcx.ty.kind() {
+                ty::Slice(ty) | ty::Array(ty, _) => {
+                    let arity = slice.arity();
+                    Fields::wildcards_from_tys(pcx.cx, (0..arity).map(|_| ty))
+                }
+                _ => bug!("bad slice pattern {:?} {:?}", constructor, pcx),
+            },
+            Bool(..)
+            | IntRange(..)
+            | F32Range(..)
+            | F64Range(..)
+            | Str(..)
+            | Opaque(..)
+            | NonExhaustive
+            | Hidden
+            | Missing { .. }
+            | Wildcard => Fields::empty(),
+            Or => {
+                bug!("called `Fields::wildcards` on an `Or` ctor")
+            }
+        };
+        debug!(?ret);
+        ret
+    }
+
+    /// Returns the list of patterns.
+    pub(super) fn iter_patterns<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
+        self.fields.iter()
+    }
+}
+
+/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
+fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
+    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
+        if let PatKind::Or { pats } = &pat.kind {
+            for pat in pats.iter() {
+                expand(pat, vec);
+            }
+        } else {
+            vec.push(pat)
+        }
+    }
+
+    let mut pats = Vec::new();
+    expand(pat, &mut pats);
+    pats
+}
+
+/// Values and patterns can be represented as a constructor applied to some fields. This represents
+/// a pattern in this form.
+/// This also uses interior mutability to keep track of whether the pattern has been found reachable
+/// during analysis. For this reason they cannot be cloned.
+/// A `DeconstructedPat` will almost always come from user input; the only exception are some
+/// `Wildcard`s introduced during specialization.
+pub struct DeconstructedPat<'p, 'tcx> {
+    ctor: Constructor<'tcx>,
+    fields: Fields<'p, 'tcx>,
+    ty: Ty<'tcx>,
+    span: Span,
+    /// Whether removing this arm would change the behavior of the match expression.
+    useful: Cell<bool>,
+}
+
+impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
+    pub(super) fn wildcard(ty: Ty<'tcx>, span: Span) -> Self {
+        Self::new(Wildcard, Fields::empty(), ty, span)
+    }
+
+    pub(super) fn new(
+        ctor: Constructor<'tcx>,
+        fields: Fields<'p, 'tcx>,
+        ty: Ty<'tcx>,
+        span: Span,
+    ) -> Self {
+        DeconstructedPat { ctor, fields, ty, span, useful: Cell::new(false) }
+    }
+
+    /// Note: the input patterns must have been lowered through
+    /// `rustc_mir_build::thir::pattern::check_match::MatchVisitor::lower_pattern`.
+    pub fn from_pat(cx: &MatchCheckCtxt<'p, 'tcx>, pat: &Pat<'tcx>) -> Self {
+        let mkpat = |pat| DeconstructedPat::from_pat(cx, pat);
+        let ctor;
+        let fields;
+        match &pat.kind {
+            PatKind::AscribeUserType { subpattern, .. }
+            | PatKind::InlineConstant { subpattern, .. } => return mkpat(subpattern),
+            PatKind::Binding { subpattern: Some(subpat), .. } => return mkpat(subpat),
+            PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
+                ctor = Wildcard;
+                fields = Fields::empty();
+            }
+            PatKind::Deref { subpattern } => {
+                ctor = Single;
+                fields = Fields::singleton(cx, mkpat(subpattern));
+            }
+            PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => {
+                match pat.ty.kind() {
+                    ty::Tuple(fs) => {
+                        ctor = Single;
+                        let mut wilds: SmallVec<[_; 2]> =
+                            fs.iter().map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
+                        for pat in subpatterns {
+                            wilds[pat.field.index()] = mkpat(&pat.pattern);
+                        }
+                        fields = Fields::from_iter(cx, wilds);
+                    }
+                    ty::Adt(adt, args) if adt.is_box() => {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
+                        // _)` or a box pattern. As a hack to avoid an ICE with the former, we
+                        // ignore other fields than the first one. This will trigger an error later
+                        // anyway.
+                        // See https://github.com/rust-lang/rust/issues/82772 ,
+                        // explanation: https://github.com/rust-lang/rust/pull/82789#issuecomment-796921977
+                        // The problem is that we can't know from the type whether we'll match
+                        // normally or through box-patterns. We'll have to figure out a proper
+                        // solution when we introduce generalized deref patterns. Also need to
+                        // prevent mixing of those two options.
+                        let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
+                        let pat = if let Some(pat) = pattern {
+                            mkpat(&pat.pattern)
+                        } else {
+                            DeconstructedPat::wildcard(args.type_at(0), pat.span)
+                        };
+                        ctor = Single;
+                        fields = Fields::singleton(cx, pat);
+                    }
+                    ty::Adt(adt, _) => {
+                        ctor = match pat.kind {
+                            PatKind::Leaf { .. } => Single,
+                            PatKind::Variant { variant_index, .. } => Variant(variant_index),
+                            _ => bug!(),
+                        };
+                        let variant = &adt.variant(ctor.variant_index_for_adt(*adt));
+                        // For each field in the variant, we store the relevant index into `self.fields` if any.
+                        let mut field_id_to_id: Vec<Option<usize>> =
+                            (0..variant.fields.len()).map(|_| None).collect();
+                        let tys = Fields::list_variant_nonhidden_fields(cx, pat.ty, variant)
+                            .enumerate()
+                            .map(|(i, (field, ty))| {
+                                field_id_to_id[field.index()] = Some(i);
+                                ty
+                            });
+                        let mut wilds: SmallVec<[_; 2]> =
+                            tys.map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
+                        for pat in subpatterns {
+                            if let Some(i) = field_id_to_id[pat.field.index()] {
+                                wilds[i] = mkpat(&pat.pattern);
+                            }
+                        }
+                        fields = Fields::from_iter(cx, wilds);
+                    }
+                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
+                }
+            }
+            PatKind::Constant { value } => {
+                match pat.ty.kind() {
+                    ty::Bool => {
+                        ctor = match value.try_eval_bool(cx.tcx, cx.param_env) {
+                            Some(b) => Bool(b),
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = Fields::empty();
+                    }
+                    ty::Char | ty::Int(_) | ty::Uint(_) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => IntRange(IntRange::from_bits(cx.tcx, pat.ty, bits)),
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = Fields::empty();
+                    }
+                    ty::Float(ty::FloatTy::F32) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => {
+                                use rustc_apfloat::Float;
+                                let value = rustc_apfloat::ieee::Single::from_bits(bits);
+                                F32Range(value, value, RangeEnd::Included)
+                            }
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = Fields::empty();
+                    }
+                    ty::Float(ty::FloatTy::F64) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => {
+                                use rustc_apfloat::Float;
+                                let value = rustc_apfloat::ieee::Double::from_bits(bits);
+                                F64Range(value, value, RangeEnd::Included)
+                            }
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = Fields::empty();
+                    }
+                    ty::Ref(_, t, _) if t.is_str() => {
+                        // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
+                        // with other `Deref` patterns. This could have been done in `const_to_pat`,
+                        // but that causes issues with the rest of the matching code.
+                        // So here, the constructor for a `"foo"` pattern is `&` (represented by
+                        // `Single`), and has one field. That field has constructor `Str(value)` and no
+                        // fields.
+                        // Note: `t` is `str`, not `&str`.
+                        let subpattern =
+                            DeconstructedPat::new(Str(*value), Fields::empty(), *t, pat.span);
+                        ctor = Single;
+                        fields = Fields::singleton(cx, subpattern)
+                    }
+                    // All constants that can be structurally matched have already been expanded
+                    // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
+                    // opaque.
+                    _ => {
+                        ctor = Opaque(OpaqueId::new());
+                        fields = Fields::empty();
+                    }
+                }
+            }
+            PatKind::Range(patrange) => {
+                let PatRange { lo, hi, end, .. } = patrange.as_ref();
+                let ty = pat.ty;
+                ctor = match ty.kind() {
+                    ty::Char | ty::Int(_) | ty::Uint(_) => {
+                        let lo =
+                            MaybeInfiniteInt::from_pat_range_bdy(*lo, ty, cx.tcx, cx.param_env);
+                        let hi =
+                            MaybeInfiniteInt::from_pat_range_bdy(*hi, ty, cx.tcx, cx.param_env);
+                        IntRange(IntRange::from_range(lo, hi, *end))
+                    }
+                    ty::Float(fty) => {
+                        use rustc_apfloat::Float;
+                        let lo = lo.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        let hi = hi.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        match fty {
+                            ty::FloatTy::F32 => {
+                                use rustc_apfloat::ieee::Single;
+                                let lo = lo.map(Single::from_bits).unwrap_or(-Single::INFINITY);
+                                let hi = hi.map(Single::from_bits).unwrap_or(Single::INFINITY);
+                                F32Range(lo, hi, *end)
+                            }
+                            ty::FloatTy::F64 => {
+                                use rustc_apfloat::ieee::Double;
+                                let lo = lo.map(Double::from_bits).unwrap_or(-Double::INFINITY);
+                                let hi = hi.map(Double::from_bits).unwrap_or(Double::INFINITY);
+                                F64Range(lo, hi, *end)
+                            }
+                        }
+                    }
+                    _ => bug!("invalid type for range pattern: {}", ty),
+                };
+                fields = Fields::empty();
+            }
+            PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
+                let array_len = match pat.ty.kind() {
+                    ty::Array(_, length) => {
+                        Some(length.eval_target_usize(cx.tcx, cx.param_env) as usize)
+                    }
+                    ty::Slice(_) => None,
+                    _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
+                };
+                let kind = if slice.is_some() {
+                    VarLen(prefix.len(), suffix.len())
+                } else {
+                    FixedLen(prefix.len() + suffix.len())
+                };
+                ctor = Slice(Slice::new(array_len, kind));
+                fields =
+                    Fields::from_iter(cx, prefix.iter().chain(suffix.iter()).map(|p| mkpat(&*p)));
+            }
+            PatKind::Or { .. } => {
+                ctor = Or;
+                let pats = expand_or_pat(pat);
+                fields = Fields::from_iter(cx, pats.into_iter().map(mkpat));
+            }
+            PatKind::Never => {
+                // FIXME(never_patterns): handle `!` in exhaustiveness. This is a sane default
+                // in the meantime.
+                ctor = Wildcard;
+                fields = Fields::empty();
+            }
+            PatKind::Error(_) => {
+                ctor = Opaque(OpaqueId::new());
+                fields = Fields::empty();
+            }
+        }
+        DeconstructedPat::new(ctor, fields, pat.ty, pat.span)
+    }
+
+    pub(super) fn is_or_pat(&self) -> bool {
+        matches!(self.ctor, Or)
+    }
+    /// Expand this (possibly-nested) or-pattern into its alternatives.
+    pub(super) fn flatten_or_pat(&'p self) -> SmallVec<[&'p Self; 1]> {
+        if self.is_or_pat() {
+            self.iter_fields().flat_map(|p| p.flatten_or_pat()).collect()
+        } else {
+            smallvec![self]
+        }
+    }
+
+    pub fn ctor(&self) -> &Constructor<'tcx> {
+        &self.ctor
+    }
+    pub fn ty(&self) -> Ty<'tcx> {
+        self.ty
+    }
+    pub fn span(&self) -> Span {
+        self.span
+    }
+
+    pub fn iter_fields<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
+        self.fields.iter_patterns()
+    }
+
+    /// Specialize this pattern with a constructor.
+    /// `other_ctor` can be different from `self.ctor`, but must be covered by it.
+    pub(super) fn specialize<'a>(
+        &'a self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        other_ctor: &Constructor<'tcx>,
+    ) -> SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]> {
+        match (&self.ctor, other_ctor) {
+            (Wildcard, _) => {
+                // We return a wildcard for each field of `other_ctor`.
+                Fields::wildcards(pcx, other_ctor).iter_patterns().collect()
+            }
+            (Slice(self_slice), Slice(other_slice))
+                if self_slice.arity() != other_slice.arity() =>
+            {
+                // The only tricky case: two slices of different arity. Since `self_slice` covers
+                // `other_slice`, `self_slice` must be `VarLen`, i.e. of the form
+                // `[prefix, .., suffix]`. Moreover `other_slice` is guaranteed to have a larger
+                // arity. So we fill the middle part with enough wildcards to reach the length of
+                // the new, larger slice.
+                match self_slice.kind {
+                    FixedLen(_) => bug!("{:?} doesn't cover {:?}", self_slice, other_slice),
+                    VarLen(prefix, suffix) => {
+                        let (ty::Slice(inner_ty) | ty::Array(inner_ty, _)) = *self.ty.kind() else {
+                            bug!("bad slice pattern {:?} {:?}", self.ctor, self.ty);
+                        };
+                        let prefix = &self.fields.fields[..prefix];
+                        let suffix = &self.fields.fields[self_slice.arity() - suffix..];
+                        let wildcard: &_ = pcx
+                            .cx
+                            .pattern_arena
+                            .alloc(DeconstructedPat::wildcard(inner_ty, DUMMY_SP));
+                        let extra_wildcards = other_slice.arity() - self_slice.arity();
+                        let extra_wildcards = (0..extra_wildcards).map(|_| wildcard);
+                        prefix.iter().chain(extra_wildcards).chain(suffix).collect()
+                    }
+                }
+            }
+            _ => self.fields.iter_patterns().collect(),
+        }
+    }
+
+    /// We keep track for each pattern if it was ever useful during the analysis. This is used
+    /// with `redundant_spans` to report redundant subpatterns arising from or patterns.
+    pub(super) fn set_useful(&self) {
+        self.useful.set(true)
+    }
+    pub(super) fn is_useful(&self) -> bool {
+        if self.useful.get() {
+            true
+        } else if self.is_or_pat() && self.iter_fields().any(|f| f.is_useful()) {
+            // We always expand or patterns in the matrix, so we will never see the actual
+            // or-pattern (the one with constructor `Or`) in the column. As such, it will not be
+            // marked as useful itself, only its children will. We recover this information here.
+            self.set_useful();
+            true
+        } else {
+            false
+        }
+    }
+
+    /// Report the spans of subpatterns that were not useful, if any.
+    pub(super) fn redundant_spans(&self) -> Vec<Span> {
+        let mut spans = Vec::new();
+        self.collect_redundant_spans(&mut spans);
+        spans
+    }
+    fn collect_redundant_spans(&self, spans: &mut Vec<Span>) {
+        // We don't look at subpatterns if we already reported the whole pattern as redundant.
+        if !self.is_useful() {
+            spans.push(self.span);
+        } else {
+            for p in self.iter_fields() {
+                p.collect_redundant_spans(spans);
+            }
+        }
+    }
+}
+
+/// This is mostly copied from the `Pat` impl. This is best effort and not good enough for a
+/// `Display` impl.
+impl<'p, 'tcx> fmt::Debug for DeconstructedPat<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Printing lists is a chore.
+        let mut first = true;
+        let mut start_or_continue = |s| {
+            if first {
+                first = false;
+                ""
+            } else {
+                s
+            }
+        };
+        let mut start_or_comma = || start_or_continue(", ");
+
+        match &self.ctor {
+            Single | Variant(_) => match self.ty.kind() {
+                ty::Adt(def, _) if def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    let subpattern = self.iter_fields().next().unwrap();
+                    write!(f, "box {subpattern:?}")
+                }
+                ty::Adt(..) | ty::Tuple(..) => {
+                    let variant = match self.ty.kind() {
+                        ty::Adt(adt, _) => Some(adt.variant(self.ctor.variant_index_for_adt(*adt))),
+                        ty::Tuple(_) => None,
+                        _ => unreachable!(),
+                    };
+
+                    if let Some(variant) = variant {
+                        write!(f, "{}", variant.name)?;
+                    }
+
+                    // Without `cx`, we can't know which field corresponds to which, so we can't
+                    // get the names of the fields. Instead we just display everything as a tuple
+                    // struct, which should be good enough.
+                    write!(f, "(")?;
+                    for p in self.iter_fields() {
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "{p:?}")?;
+                    }
+                    write!(f, ")")
+                }
+                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+                // be careful to detect strings here. However a string literal pattern will never
+                // be reported as a non-exhaustiveness witness, so we can ignore this issue.
+                ty::Ref(_, _, mutbl) => {
+                    let subpattern = self.iter_fields().next().unwrap();
+                    write!(f, "&{}{:?}", mutbl.prefix_str(), subpattern)
+                }
+                _ => write!(f, "_"),
+            },
+            Slice(slice) => {
+                let mut subpatterns = self.fields.iter_patterns();
+                write!(f, "[")?;
+                match slice.kind {
+                    FixedLen(_) => {
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                    VarLen(prefix_len, _) => {
+                        for p in subpatterns.by_ref().take(prefix_len) {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "..")?;
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                }
+                write!(f, "]")
+            }
+            Bool(b) => write!(f, "{b}"),
+            // Best-effort, will render signed ranges incorrectly
+            IntRange(range) => write!(f, "{range:?}"),
+            F32Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
+            F64Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
+            Str(value) => write!(f, "{value}"),
+            Opaque(..) => write!(f, "<constant pattern>"),
+            Or => {
+                for pat in self.iter_fields() {
+                    write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
+                }
+                Ok(())
+            }
+            Wildcard | Missing { .. } | NonExhaustive | Hidden => write!(f, "_ : {:?}", self.ty),
+        }
+    }
+}
+
+/// Same idea as `DeconstructedPat`, except this is a fictitious pattern built up for diagnostics
+/// purposes. As such they don't use interning and can be cloned.
+#[derive(Debug, Clone)]
+pub struct WitnessPat<'tcx> {
+    ctor: Constructor<'tcx>,
+    pub(crate) fields: Vec<WitnessPat<'tcx>>,
+    ty: Ty<'tcx>,
+}
+
+impl<'tcx> WitnessPat<'tcx> {
+    pub(super) fn new(ctor: Constructor<'tcx>, fields: Vec<Self>, ty: Ty<'tcx>) -> Self {
+        Self { ctor, fields, ty }
+    }
+    pub(super) fn wildcard(ty: Ty<'tcx>) -> Self {
+        Self::new(Wildcard, Vec::new(), ty)
+    }
+
+    /// Construct a pattern that matches everything that starts with this constructor.
+    /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
+    /// `Some(_)`.
+    pub(super) fn wild_from_ctor(pcx: &PatCtxt<'_, '_, 'tcx>, ctor: Constructor<'tcx>) -> Self {
+        // Reuse `Fields::wildcards` to get the types.
+        let fields = Fields::wildcards(pcx, &ctor)
+            .iter_patterns()
+            .map(|deco_pat| Self::wildcard(deco_pat.ty()))
+            .collect();
+        Self::new(ctor, fields, pcx.ty)
+    }
+
+    pub fn ctor(&self) -> &Constructor<'tcx> {
+        &self.ctor
+    }
+    pub fn ty(&self) -> Ty<'tcx> {
+        self.ty
+    }
+
+    /// Convert back to a `thir::Pat` for diagnostic purposes. This panics for patterns that don't
+    /// appear in diagnostics, like float ranges.
+    pub fn to_diagnostic_pat(&self, cx: &MatchCheckCtxt<'_, 'tcx>) -> Pat<'tcx> {
+        let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
+        let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_diagnostic_pat(cx)));
+        let kind = match &self.ctor {
+            Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
+            IntRange(range) => return range.to_diagnostic_pat(self.ty, cx.tcx),
+            Single | Variant(_) => match self.ty.kind() {
+                ty::Tuple(..) => PatKind::Leaf {
+                    subpatterns: subpatterns
+                        .enumerate()
+                        .map(|(i, pattern)| FieldPat { field: FieldIdx::new(i), pattern })
+                        .collect(),
+                },
+                ty::Adt(adt_def, _) if adt_def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    PatKind::Deref { subpattern: subpatterns.next().unwrap() }
+                }
+                ty::Adt(adt_def, args) => {
+                    let variant_index = self.ctor.variant_index_for_adt(*adt_def);
+                    let variant = &adt_def.variant(variant_index);
+                    let subpatterns = Fields::list_variant_nonhidden_fields(cx, self.ty, variant)
+                        .zip(subpatterns)
+                        .map(|((field, _ty), pattern)| FieldPat { field, pattern })
+                        .collect();
+
+                    if adt_def.is_enum() {
+                        PatKind::Variant { adt_def: *adt_def, args, variant_index, subpatterns }
+                    } else {
+                        PatKind::Leaf { subpatterns }
+                    }
+                }
+                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+                // be careful to reconstruct the correct constant pattern here. However a string
+                // literal pattern will never be reported as a non-exhaustiveness witness, so we
+                // ignore this issue.
+                ty::Ref(..) => PatKind::Deref { subpattern: subpatterns.next().unwrap() },
+                _ => bug!("unexpected ctor for type {:?} {:?}", self.ctor, self.ty),
+            },
+            Slice(slice) => {
+                match slice.kind {
+                    FixedLen(_) => PatKind::Slice {
+                        prefix: subpatterns.collect(),
+                        slice: None,
+                        suffix: Box::new([]),
+                    },
+                    VarLen(prefix, _) => {
+                        let mut subpatterns = subpatterns.peekable();
+                        let mut prefix: Vec<_> = subpatterns.by_ref().take(prefix).collect();
+                        if slice.array_len.is_some() {
+                            // Improves diagnostics a bit: if the type is a known-size array, instead
+                            // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
+                            // This is incorrect if the size is not known, since `[_, ..]` captures
+                            // arrays of lengths `>= 1` whereas `[..]` captures any length.
+                            while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
+                                prefix.pop();
+                            }
+                            while subpatterns.peek().is_some()
+                                && is_wildcard(subpatterns.peek().unwrap())
+                            {
+                                subpatterns.next();
+                            }
+                        }
+                        let suffix: Box<[_]> = subpatterns.collect();
+                        let wild = Pat::wildcard_from_ty(self.ty);
+                        PatKind::Slice {
+                            prefix: prefix.into_boxed_slice(),
+                            slice: Some(Box::new(wild)),
+                            suffix,
+                        }
+                    }
+                }
+            }
+            &Str(value) => PatKind::Constant { value },
+            Wildcard | NonExhaustive | Hidden => PatKind::Wild,
+            Missing { .. } => bug!(
+                "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
+                `Missing` should have been processed in `apply_constructors`"
+            ),
+            F32Range(..) | F64Range(..) | Opaque(..) | Or => {
+                bug!("can't convert to pattern: {:?}", self)
+            }
+        };
+
+        Pat { ty: self.ty, span: DUMMY_SP, kind }
+    }
+
+    pub fn iter_fields<'a>(&'a self) -> impl Iterator<Item = &'a WitnessPat<'tcx>> {
+        self.fields.iter()
+    }
+}
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
new file mode 100644
index 00000000000..5554f3fc36c
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -0,0 +1,1646 @@
+//! # Match exhaustiveness and redundancy algorithm
+//!
+//! This file contains the logic for exhaustiveness and usefulness checking for pattern-matching.
+//! Specifically, given a list of patterns in a match, we can tell whether:
+//! (a) a given pattern is redundant
+//! (b) the patterns cover every possible value for the type (exhaustiveness)
+//!
+//! The algorithm implemented here is inspired from the one described in [this
+//! paper](http://moscova.inria.fr/~maranget/papers/warn/index.html). We have however changed it in
+//! various ways to accommodate the variety of patterns that Rust supports. We thus explain our
+//! version here, without being as precise.
+//!
+//! Fun fact: computing exhaustiveness is NP-complete, because we can encode a SAT problem as an
+//! exhaustiveness problem. See [here](https://niedzejkob.p4.team/rust-np) for the fun details.
+//!
+//!
+//! # Summary
+//!
+//! The algorithm is given as input a list of patterns, one for each arm of a match, and computes
+//! the following:
+//! - a set of values that match none of the patterns (if any),
+//! - for each subpattern (taking into account or-patterns), whether removing it would change
+//!     anything about how the match executes, i.e. whether it is useful/not redundant.
+//!
+//! To a first approximation, the algorithm works by exploring all possible values for the type
+//! being matched on, and determining which arm(s) catch which value. To make this tractable we
+//! cleverly group together values, as we'll see below.
+//!
+//! The entrypoint of this file is the [`compute_match_usefulness`] function, which computes
+//! usefulness for each subpattern and exhaustiveness for the whole match.
+//!
+//! In this page we explain the necessary concepts to understand how the algorithm works.
+//!
+//!
+//! # Usefulness
+//!
+//! The central concept of this file is the notion of "usefulness". Given some patterns `p_1 ..
+//! p_n`, a pattern `q` is said to be *useful* if there is a value that is matched by `q` and by
+//! none of the `p_i`. We write `usefulness(p_1 .. p_n, q)` for a function that returns a list of
+//! such values. The aim of this file is to compute it efficiently.
+//!
+//! This is enough to compute usefulness: a pattern in a `match` expression is redundant iff it is
+//! not useful w.r.t. the patterns above it:
+//! ```compile_fail,E0004
+//! # #![feature(exclusive_range_pattern)]
+//! # fn foo() {
+//! match Some(0u32) {
+//!     Some(0..100) => {},
+//!     Some(90..190) => {}, // useful: `Some(150)` is matched by this but not the branch above
+//!     Some(50..150) => {}, // redundant: all the values this matches are already matched by
+//!                          //   the branches above
+//!     None => {},          // useful: `None` is matched by this but not the branches above
+//! }
+//! # }
+//! ```
+//!
+//! This is also enough to compute exhaustiveness: a match is exhaustive iff the wildcard `_`
+//! pattern is _not_ useful w.r.t. the patterns in the match. The values returned by `usefulness`
+//! are used to tell the user which values are missing.
+//! ```compile_fail,E0004
+//! # fn foo(x: Option<u32>) {
+//! match x {
+//!     None => {},
+//!     Some(0) => {},
+//!     // not exhaustive: `_` is useful because it matches `Some(1)`
+//! }
+//! # }
+//! ```
+//!
+//!
+//! # Constructors and fields
+//!
+//! In the value `Pair(Some(0), true)`, `Pair` is called the constructor of the value, and `Some(0)`
+//! and `true` are its fields. Every matcheable value can be decomposed in this way. Examples of
+//! constructors are: `Some`, `None`, `(,)` (the 2-tuple constructor), `Foo {..}` (the constructor
+//! for a struct `Foo`), and `2` (the constructor for the number `2`).
+//!
+//! Each constructor takes a fixed number of fields; this is called its arity. `Pair` and `(,)` have
+//! arity 2, `Some` has arity 1, `None` and `42` have arity 0. Each type has a known set of
+//! constructors. Some types have many constructors (like `u64`) or even an infinitely many (like
+//! `&str` and `&[T]`).
+//!
+//! Patterns are similar: `Pair(Some(_), _)` has constructor `Pair` and two fields. The difference
+//! is that we get some extra pattern-only constructors, namely: the wildcard `_`, variable
+//! bindings, integer ranges like `0..=10`, and variable-length slices like `[_, .., _]`. We treat
+//! or-patterns separately, see the dedicated section below.
+//!
+//! Now to check if a value `v` matches a pattern `p`, we check if `v`'s constructor matches `p`'s
+//! constructor, then recursively compare their fields if necessary. A few representative examples:
+//!
+//! - `matches!(v, _) := true`
+//! - `matches!((v0,  v1), (p0,  p1)) := matches!(v0, p0) && matches!(v1, p1)`
+//! - `matches!(Foo { bar: v0, baz: v1 }, Foo { bar: p0, baz: p1 }) := matches!(v0, p0) && matches!(v1, p1)`
+//! - `matches!(Ok(v0), Ok(p0)) := matches!(v0, p0)`
+//! - `matches!(Ok(v0), Err(p0)) := false` (incompatible variants)
+//! - `matches!(v, 1..=100) := matches!(v, 1) || ... || matches!(v, 100)`
+//! - `matches!([v0], [p0, .., p1]) := false` (incompatible lengths)
+//! - `matches!([v0, v1, v2], [p0, .., p1]) := matches!(v0, p0) && matches!(v2, p1)`
+//!
+//! Constructors, fields and relevant operations are defined in the [`super::deconstruct_pat`]
+//! module. The question of whether a constructor is matched by another one is answered by
+//! [`Constructor::is_covered_by`].
+//!
+//! Note 1: variable bindings (like the `x` in `Some(x)`) match anything, so we treat them as wildcards.
+//! Note 2: this only applies to matcheable values. For example a value of type `Rc<u64>` can't be
+//! deconstructed that way.
+//!
+//!
+//!
+//! # Specialization
+//!
+//! The examples in the previous section motivate the operation at the heart of the algorithm:
+//! "specialization". It captures this idea of "removing one layer of constructor".
+//!
+//! `specialize(c, p)` takes a value-only constructor `c` and a pattern `p`, and returns a
+//! pattern-tuple or nothing. It works as follows:
+//!
+//! - Specializing for the wrong constructor returns nothing
+//!
+//!   - `specialize(None, Some(p0)) := <nothing>`
+//!   - `specialize([,,,], [p0]) := <nothing>`
+//!
+//! - Specializing for the correct constructor returns a tuple of the fields
+//!
+//!   - `specialize(Variant1, Variant1(p0, p1, p2)) := (p0, p1, p2)`
+//!   - `specialize(Foo{ bar, baz, quz }, Foo { bar: p0, baz: p1, .. }) := (p0, p1, _)`
+//!   - `specialize([,,,], [p0, .., p1]) := (p0, _, _, p1)`
+//!
+//! We get the following property: for any values `v_1, .., v_n` of appropriate types, we have:
+//! ```text
+//! matches!(c(v_1, .., v_n), p)
+//! <=> specialize(c, p) returns something
+//!     && matches!((v_1, .., v_n), specialize(c, p))
+//! ```
+//!
+//! We also extend specialization to pattern-tuples by applying it to the first pattern:
+//! `specialize(c, (p_0, .., p_n)) := specialize(c, p_0) ++ (p_1, .., p_m)`
+//! where `++` is concatenation of tuples.
+//!
+//!
+//! The previous property extends to pattern-tuples:
+//! ```text
+//! matches!((c(v_1, .., v_n), w_1, .., w_m), (p_0, p_1, .., p_m))
+//! <=> specialize(c, p_0) does not error
+//!     && matches!((v_1, .., v_n, w_1, .., w_m), specialize(c, (p_0, p_1, .., p_m)))
+//! ```
+//!
+//! Whether specialization returns something or not is given by [`Constructor::is_covered_by`].
+//! Specialization of a pattern is computed in [`DeconstructedPat::specialize`]. Specialization for
+//! a pattern-tuple is computed in [`PatStack::pop_head_constructor`]. Finally, specialization for a
+//! set of pattern-tuples is computed in [`Matrix::specialize_constructor`].
+//!
+//!
+//!
+//! # Undoing specialization
+//!
+//! To construct witnesses we will need an inverse of specialization. If `c` is a constructor of
+//! arity `n`, we define `unspecialize` as:
+//! `unspecialize(c, (p_1, .., p_n, q_1, .., q_m)) := (c(p_1, .., p_n), q_1, .., q_m)`.
+//!
+//! This is done for a single witness-tuple in [`WitnessStack::apply_constructor`], and for a set of
+//! witness-tuples in [`WitnessMatrix::apply_constructor`].
+//!
+//!
+//!
+//! # Computing usefulness
+//!
+//! We now present a naive version of the algorithm for computing usefulness. From now on we operate
+//! on pattern-tuples.
+//!
+//! Let `pt_1, .., pt_n` and `qt` be length-m tuples of patterns for the same type `(T_1, .., T_m)`.
+//! We compute `usefulness(tp_1, .., tp_n, tq)` as follows:
+//!
+//! - Base case: `m == 0`.
+//!     The pattern-tuples are all empty, i.e. they're all `()`. Thus `tq` is useful iff there are
+//!     no rows above it, i.e. if `n == 0`. In that case we return `()` as a witness-tuple of
+//!     usefulness of `tq`.
+//!
+//! - Inductive case: `m > 0`.
+//!     In this naive version, we list all the possible constructors for values of type `T1` (we
+//!     will be more clever in the next section).
+//!
+//!     - For each such constructor `c` for which `specialize(c, tq)` is not nothing:
+//!         - We recursively compute `usefulness(specialize(c, tp_1) ... specialize(c, tp_n), specialize(c, tq))`,
+//!             where we discard any `specialize(c, p_i)` that returns nothing.
+//!         - For each witness-tuple `w` found, we apply `unspecialize(c, w)` to it.
+//!
+//!     - We return the all the witnesses found, if any.
+//!
+//!
+//! Let's take the following example:
+//! ```compile_fail,E0004
+//! # enum Enum { Variant1(()), Variant2(Option<bool>, u32)}
+//! # use Enum::*;
+//! # fn foo(x: Enum) {
+//! match x {
+//!     Variant1(_) => {} // `p1`
+//!     Variant2(None, 0) => {} // `p2`
+//!     Variant2(Some(_), 0) => {} // `q`
+//! }
+//! # }
+//! ```
+//!
+//! To compute the usefulness of `q`, we would proceed as follows:
+//! ```text
+//! Start:
+//!   `tp1 = [Variant1(_)]`
+//!   `tp2 = [Variant2(None, 0)]`
+//!   `tq  = [Variant2(Some(true), 0)]`
+//!
+//!   Constructors are `Variant1` and `Variant2`. Only `Variant2` can specialize `tq`.
+//!   Specialize with `Variant2`:
+//!     `tp2 = [None, 0]`
+//!     `tq  = [Some(true), 0]`
+//!
+//!     Constructors are `None` and `Some`. Only `Some` can specialize `tq`.
+//!     Specialize with `Some`:
+//!       `tq  = [true, 0]`
+//!
+//!       Constructors are `false` and `true`. Only `true` can specialize `tq`.
+//!       Specialize with `true`:
+//!         `tq  = [0]`
+//!
+//!         Constructors are `0`, `1`, .. up to infinity. Only `0` can specialize `tq`.
+//!         Specialize with `0`:
+//!           `tq  = []`
+//!
+//!           m == 0 and n == 0, so `tq` is useful with witness `[]`.
+//!             `witness  = []`
+//!
+//!         Unspecialize with `0`:
+//!           `witness  = [0]`
+//!       Unspecialize with `true`:
+//!         `witness  = [true, 0]`
+//!     Unspecialize with `Some`:
+//!       `witness  = [Some(true), 0]`
+//!   Unspecialize with `Variant2`:
+//!     `witness  = [Variant2(Some(true), 0)]`
+//! ```
+//!
+//! Therefore `usefulness(tp_1, tp_2, tq)` returns the single witness-tuple `[Variant2(Some(true), 0)]`.
+//!
+//!
+//! Computing the set of constructors for a type is done in [`ConstructorSet::for_ty`]. See the
+//! following sections for more accurate versions of the algorithm and corresponding links.
+//!
+//!
+//!
+//! # Computing usefulness and exhaustiveness in one go
+//!
+//! The algorithm we have described so far computes usefulness of each pattern in turn, and ends by
+//! checking if `_` is useful to determine exhaustiveness of the whole match. In practice, instead
+//! of doing "for each pattern { for each constructor { ... } }", we do "for each constructor { for
+//! each pattern { ... } }". This allows us to compute everything in one go.
+//!
+//! [`Matrix`] stores the set of pattern-tuples under consideration. We track usefulness of each
+//! row mutably in the matrix as we go along. We ignore witnesses of usefulness of the match rows.
+//! We gather witnesses of the usefulness of `_` in [`WitnessMatrix`]. The algorithm that computes
+//! all this is in [`compute_exhaustiveness_and_usefulness`].
+//!
+//! See the full example at the bottom of this documentation.
+//!
+//!
+//!
+//! # Making usefulness tractable: constructor splitting
+//!
+//! We're missing one last detail: which constructors do we list? Naively listing all value
+//! constructors cannot work for types like `u64` or `&str`, so we need to be more clever. The final
+//! clever idea for this algorithm is that we can group together constructors that behave the same.
+//!
+//! Examples:
+//! ```compile_fail,E0004
+//! match (0, false) {
+//!     (0 ..=100, true) => {}
+//!     (50..=150, false) => {}
+//!     (0 ..=200, _) => {}
+//! }
+//! ```
+//!
+//! In this example, trying any of `0`, `1`, .., `49` will give the same specialized matrix, and
+//! thus the same usefulness/exhaustiveness results. We can thus accelerate the algorithm by
+//! trying them all at once. Here in fact, the only cases we need to consider are: `0..50`,
+//! `50..=100`, `101..=150`,`151..=200` and `201..`.
+//!
+//! ```
+//! enum Direction { North, South, East, West }
+//! # let wind = (Direction::North, 0u8);
+//! match wind {
+//!     (Direction::North, 50..) => {}
+//!     (_, _) => {}
+//! }
+//! ```
+//!
+//! In this example, trying any of `South`, `East`, `West` will give the same specialized matrix. By
+//! the same reasoning, we only need to try two cases: `North`, and "everything else".
+//!
+//! We call _constructor splitting_ the operation that computes such a minimal set of cases to try.
+//! This is done in [`ConstructorSet::split`] and explained in [`super::deconstruct_pat`].
+//!
+//!
+//!
+//! # Or-patterns
+//!
+//! What we have described so far works well if there are no or-patterns. To handle them, if the
+//! first pattern of a row in the matrix is an or-pattern, we expand it by duplicating the rest of
+//! the row as necessary. This is handled automatically in [`Matrix`].
+//!
+//! This makes usefulness tracking subtle, because we also want to compute whether an alternative
+//! of an or-pattern is redundant, e.g. in `Some(_) | Some(0)`. We track usefulness of each
+//! subpattern by interior mutability in [`DeconstructedPat`] with `set_useful`/`is_useful`.
+//!
+//! It's unfortunate that we have to use interior mutability, but believe me (Nadrieril), I have
+//! tried [other](https://github.com/rust-lang/rust/pull/80104)
+//! [solutions](https://github.com/rust-lang/rust/pull/80632) and nothing is remotely as simple.
+//!
+//!
+//!
+//! # Constants and opaques
+//!
+//! There are two kinds of constants in patterns:
+//!
+//! * literals (`1`, `true`, `"foo"`)
+//! * named or inline consts (`FOO`, `const { 5 + 6 }`)
+//!
+//! The latter are converted into the corresponding patterns by a previous phase. For example
+//! `const_to_pat(const { [1, 2, 3] })` becomes an `Array(vec![Const(1), Const(2), Const(3)])`
+//! pattern. This gets problematic when comparing the constant via `==` would behave differently
+//! from matching on the constant converted to a pattern. The situation around this is currently
+//! unclear and the lang team is working on clarifying what we want to do there. In any case, there
+//! are constants we will not turn into patterns. We capture these with `Constructor::Opaque`. These
+//! `Opaque` patterns do not participate in exhaustiveness, specialization or overlap checking.
+//!
+//!
+//!
+//! # Usefulness vs reachability, validity, and empty patterns
+//!
+//! This is likely the subtlest aspect of the algorithm. To be fully precise, a match doesn't
+//! operate on a value, it operates on a place. In certain unsafe circumstances, it is possible for
+//! a place to not contain valid data for its type. This has subtle consequences for empty types.
+//! Take the following:
+//!
+//! ```rust
+//! enum Void {}
+//! let x: u8 = 0;
+//! let ptr: *const Void = &x as *const u8 as *const Void;
+//! unsafe {
+//!     match *ptr {
+//!         _ => println!("Reachable!"),
+//!     }
+//! }
+//! ```
+//!
+//! In this example, `ptr` is a valid pointer pointing to a place with invalid data. The `_` pattern
+//! does not look at the contents of `*ptr`, so this is ok and the arm is taken. In other words,
+//! despite the place we are inspecting being of type `Void`, there is a reachable arm. If the
+//! arm had a binding however:
+//!
+//! ```rust
+//! # #[derive(Copy, Clone)]
+//! # enum Void {}
+//! # let x: u8 = 0;
+//! # let ptr: *const Void = &x as *const u8 as *const Void;
+//! # unsafe {
+//! match *ptr {
+//!     _a => println!("Unreachable!"),
+//! }
+//! # }
+//! ```
+//!
+//! Here the binding loads the value of type `Void` from the `*ptr` place. In this example, this
+//! causes UB since the data is not valid. In the general case, this asserts validity of the data at
+//! `*ptr`. Either way, this arm will never be taken.
+//!
+//! Finally, let's consider the empty match `match *ptr {}`. If we consider this exhaustive, then
+//! having invalid data at `*ptr` is invalid. In other words, the empty match is semantically
+//! equivalent to the `_a => ...` match. In the interest of explicitness, we prefer the case with an
+//! arm, hence we won't tell the user to remove the `_a` arm. In other words, the `_a` arm is
+//! unreachable yet not redundant. This is why we lint on redundant arms rather than unreachable
+//! arms, despite the fact that the lint says "unreachable".
+//!
+//! These considerations only affects certain places, namely those that can contain non-valid data
+//! without UB. These are: pointer dereferences, reference dereferences, and union field accesses.
+//! We track in the algorithm whether a given place is known to contain valid data. This is done
+//! first by inspecting the scrutinee syntactically (which gives us `cx.known_valid_scrutinee`), and
+//! then by tracking validity of each column of the matrix (which correspond to places) as we
+//! recurse into subpatterns. That second part is done through [`ValidityConstraint`], most notably
+//! [`ValidityConstraint::specialize`].
+//!
+//! Having said all that, in practice we don't fully follow what's been presented in this section.
+//! Under `exhaustive_patterns`, we allow omitting empty arms even in `!known_valid` places, for
+//! backwards-compatibility until we have a better alternative. Without `exhaustive_patterns`, we
+//! mostly treat empty types as inhabited, except specifically a non-nested `!` or empty enum. In
+//! this specific case we also allow the empty match regardless of place validity, for
+//! backwards-compatibility. Hopefully we can eventually deprecate this.
+//!
+//!
+//!
+//! # Full example
+//!
+//! We illustrate a full run of the algorithm on the following match.
+//!
+//! ```compile_fail,E0004
+//! # struct Pair(Option<u32>, bool);
+//! # fn foo(x: Pair) -> u32 {
+//! match x {
+//!     Pair(Some(0), _) => 1,
+//!     Pair(_, false) => 2,
+//!     Pair(Some(0), false) => 3,
+//! }
+//! # }
+//! ```
+//!
+//! We keep track of the original row for illustration purposes, this is not what the algorithm
+//! actually does (it tracks usefulness as a boolean on each row).
+//!
+//! ```text
+//!  ┐ Patterns:
+//!  │   1. `[Pair(Some(0), _)]`
+//!  │   2. `[Pair(_, false)]`
+//!  │   3. `[Pair(Some(0), false)]`
+//!  │
+//!  │ Specialize with `Pair`:
+//!  ├─┐ Patterns:
+//!  │ │   1. `[Some(0), _]`
+//!  │ │   2. `[_, false]`
+//!  │ │   3. `[Some(0), false]`
+//!  │ │
+//!  │ │ Specialize with `Some`:
+//!  │ ├─┐ Patterns:
+//!  │ │ │   1. `[0, _]`
+//!  │ │ │   2. `[_, false]`
+//!  │ │ │   3. `[0, false]`
+//!  │ │ │
+//!  │ │ │ Specialize with `0`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   1. `[_]`
+//!  │ │ │ │   3. `[false]`
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `true`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   1. `[]`
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), true)`).
+//!  │ │ │ ├─┘
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `false`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   1. `[]`
+//!  │ │ │ │ │   3. `[]`
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), false)`).
+//!  │ │ │ ├─┘
+//!  │ │ ├─┘
+//!  │ │ │
+//!  │ │ │ Specialize with `1..`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   2. `[false]`
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `true`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   // no rows left
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We have found an unmatched value (`Pair(Some(1..), true)`)! This gives us a witness.
+//!  │ │ │ │ │ New witnesses:
+//!  │ │ │ │ │   `[]`
+//!  │ │ │ ├─┘
+//!  │ │ │ │ Unspecialize new witnesses with `true`:
+//!  │ │ │ │   `[true]`
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `false`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   2. `[]`
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We note arm 2 is useful (by `Pair(Some(1..), false)`).
+//!  │ │ │ ├─┘
+//!  │ │ │ │
+//!  │ │ │ │ Total witnesses for `1..`:
+//!  │ │ │ │   `[true]`
+//!  │ │ ├─┘
+//!  │ │ │ Unspecialize new witnesses with `1..`:
+//!  │ │ │   `[1.., true]`
+//!  │ │ │
+//!  │ │ │ Total witnesses for `Some`:
+//!  │ │ │   `[1.., true]`
+//!  │ ├─┘
+//!  │ │ Unspecialize new witnesses with `Some`:
+//!  │ │   `[Some(1..), true]`
+//!  │ │
+//!  │ │ Specialize with `None`:
+//!  │ ├─┐ Patterns:
+//!  │ │ │   2. `[false]`
+//!  │ │ │
+//!  │ │ │ Specialize with `true`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   // no rows left
+//!  │ │ │ │
+//!  │ │ │ │ We have found an unmatched value (`Pair(None, true)`)! This gives us a witness.
+//!  │ │ │ │ New witnesses:
+//!  │ │ │ │   `[]`
+//!  │ │ ├─┘
+//!  │ │ │ Unspecialize new witnesses with `true`:
+//!  │ │ │   `[true]`
+//!  │ │ │
+//!  │ │ │ Specialize with `false`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   2. `[]`
+//!  │ │ │ │
+//!  │ │ │ │ We note arm 2 is useful (by `Pair(None, false)`).
+//!  │ │ ├─┘
+//!  │ │ │
+//!  │ │ │ Total witnesses for `None`:
+//!  │ │ │   `[true]`
+//!  │ ├─┘
+//!  │ │ Unspecialize new witnesses with `None`:
+//!  │ │   `[None, true]`
+//!  │ │
+//!  │ │ Total witnesses for `Pair`:
+//!  │ │   `[Some(1..), true]`
+//!  │ │   `[None, true]`
+//!  ├─┘
+//!  │ Unspecialize new witnesses with `Pair`:
+//!  │   `[Pair(Some(1..), true)]`
+//!  │   `[Pair(None, true)]`
+//!  │
+//!  │ Final witnesses:
+//!  │   `[Pair(Some(1..), true)]`
+//!  │   `[Pair(None, true)]`
+//!  ┘
+//! ```
+//!
+//! We conclude:
+//! - Arm 3 is redundant (it was never marked as useful);
+//! - The match is not exhaustive;
+//! - Adding arms with `Pair(Some(1..), true)` and `Pair(None, true)` would make the match exhaustive.
+//!
+//! Note that when we're deep in the algorithm, we don't know what specialization steps got us here.
+//! We can only figure out what our witnesses correspond to by unspecializing back up the stack.
+//!
+//!
+//! # Tests
+//!
+//! Note: tests specific to this file can be found in:
+//!
+//!   - `ui/pattern/usefulness`
+//!   - `ui/or-patterns`
+//!   - `ui/consts/const_in_pattern`
+//!   - `ui/rfc-2008-non-exhaustive`
+//!   - `ui/half-open-range-patterns`
+//!   - probably many others
+//!
+//! I (Nadrieril) prefer to put new tests in `ui/pattern/usefulness` unless there's a specific
+//! reason not to, for example if they crucially depend on a particular feature like `or_patterns`.
+
+use self::ValidityConstraint::*;
+use crate::constructor::{
+    Constructor, ConstructorSet, IntRange, MaybeInfiniteInt, SplitConstructorSet,
+};
+use crate::errors::{
+    NonExhaustiveOmittedPattern, NonExhaustiveOmittedPatternLintOnArm, Overlap,
+    OverlappingRangeEndpoints, Uncovered,
+};
+use crate::pat::{DeconstructedPat, WitnessPat};
+
+use rustc_arena::TypedArena;
+use rustc_data_structures::{captures::Captures, stack::ensure_sufficient_stack};
+use rustc_hir::def_id::DefId;
+use rustc_hir::HirId;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_session::lint;
+use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
+use rustc_span::{Span, DUMMY_SP};
+
+use smallvec::{smallvec, SmallVec};
+use std::fmt;
+
+pub struct MatchCheckCtxt<'p, 'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    /// The module in which the match occurs. This is necessary for
+    /// checking inhabited-ness of types because whether a type is (visibly)
+    /// inhabited can depend on whether it was defined in the current module or
+    /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
+    /// outside its module and should not be matchable with an empty match statement.
+    pub module: DefId,
+    pub param_env: ty::ParamEnv<'tcx>,
+    pub pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+    /// Lint level at the match.
+    pub match_lint_level: HirId,
+    /// The span of the whole match, if applicable.
+    pub whole_match_span: Option<Span>,
+    /// Span of the scrutinee.
+    pub scrut_span: Span,
+    /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
+    pub refutable: bool,
+    /// Whether the data at the scrutinee is known to be valid. This is false if the scrutinee comes
+    /// from a union field, a pointer deref, or a reference deref (pending opsem decisions).
+    pub known_valid_scrutinee: bool,
+}
+
+impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
+    pub(super) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
+        !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
+    }
+
+    /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
+    pub fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
+        match ty.kind() {
+            ty::Adt(def, ..) => {
+                def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did().is_local()
+            }
+            _ => false,
+        }
+    }
+}
+
+#[derive(Copy, Clone)]
+pub(super) struct PatCtxt<'a, 'p, 'tcx> {
+    pub(super) cx: &'a MatchCheckCtxt<'p, 'tcx>,
+    /// Type of the current column under investigation.
+    pub(super) ty: Ty<'tcx>,
+    /// Whether the current pattern is the whole pattern as found in a match arm, or if it's a
+    /// subpattern.
+    pub(super) is_top_level: bool,
+}
+
+impl<'a, 'p, 'tcx> PatCtxt<'a, 'p, 'tcx> {
+    /// A `PatCtxt` when code other than `is_useful` needs one.
+    fn new_dummy(cx: &'a MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
+        PatCtxt { cx, ty, is_top_level: false }
+    }
+}
+
+impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PatCtxt").field("ty", &self.ty).finish()
+    }
+}
+
+/// Serves two purposes:
+/// - in a wildcard, tracks whether the wildcard matches only valid values (i.e. is a binding `_a`)
+///     or also invalid values (i.e. is a true `_` pattern).
+/// - in the matrix, track whether a given place (aka column) is known to contain a valid value or
+///     not.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(super) enum ValidityConstraint {
+    ValidOnly,
+    MaybeInvalid,
+    /// Option for backwards compatibility: the place is not known to be valid but we allow omitting
+    /// `useful && !reachable` arms anyway.
+    MaybeInvalidButAllowOmittingArms,
+}
+
+impl ValidityConstraint {
+    pub(super) fn from_bool(is_valid_only: bool) -> Self {
+        if is_valid_only { ValidOnly } else { MaybeInvalid }
+    }
+
+    fn allow_omitting_side_effecting_arms(self) -> Self {
+        match self {
+            MaybeInvalid | MaybeInvalidButAllowOmittingArms => MaybeInvalidButAllowOmittingArms,
+            // There are no side-effecting empty arms here, nothing to do.
+            ValidOnly => ValidOnly,
+        }
+    }
+
+    pub(super) fn is_known_valid(self) -> bool {
+        matches!(self, ValidOnly)
+    }
+    pub(super) fn allows_omitting_empty_arms(self) -> bool {
+        matches!(self, ValidOnly | MaybeInvalidButAllowOmittingArms)
+    }
+
+    /// If the place has validity given by `self` and we read that the value at the place has
+    /// constructor `ctor`, this computes what we can assume about the validity of the constructor
+    /// fields.
+    ///
+    /// Pending further opsem decisions, the current behavior is: validity is preserved, except
+    /// inside `&` and union fields where validity is reset to `MaybeInvalid`.
+    pub(super) fn specialize<'tcx>(
+        self,
+        pcx: &PatCtxt<'_, '_, 'tcx>,
+        ctor: &Constructor<'tcx>,
+    ) -> Self {
+        // We preserve validity except when we go inside a reference or a union field.
+        if matches!(ctor, Constructor::Single)
+            && (matches!(pcx.ty.kind(), ty::Ref(..))
+                || matches!(pcx.ty.kind(), ty::Adt(def, ..) if def.is_union()))
+        {
+            // Validity of `x: &T` does not imply validity of `*x: T`.
+            MaybeInvalid
+        } else {
+            self
+        }
+    }
+}
+
+impl fmt::Display for ValidityConstraint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let s = match self {
+            ValidOnly => "✓",
+            MaybeInvalid | MaybeInvalidButAllowOmittingArms => "?",
+        };
+        write!(f, "{s}")
+    }
+}
+
+/// Represents a pattern-tuple under investigation.
+#[derive(Clone)]
+struct PatStack<'p, 'tcx> {
+    // Rows of len 1 are very common, which is why `SmallVec[_; 2]` works well.
+    pats: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>,
+}
+
+impl<'p, 'tcx> PatStack<'p, 'tcx> {
+    fn from_pattern(pat: &'p DeconstructedPat<'p, 'tcx>) -> Self {
+        PatStack { pats: smallvec![pat] }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.pats.is_empty()
+    }
+
+    fn len(&self) -> usize {
+        self.pats.len()
+    }
+
+    fn head(&self) -> &'p DeconstructedPat<'p, 'tcx> {
+        self.pats[0]
+    }
+
+    fn iter(&self) -> impl Iterator<Item = &DeconstructedPat<'p, 'tcx>> {
+        self.pats.iter().copied()
+    }
+
+    // Recursively expand the first or-pattern into its subpatterns. Only useful if the pattern is
+    // an or-pattern. Panics if `self` is empty.
+    fn expand_or_pat<'a>(&'a self) -> impl Iterator<Item = PatStack<'p, 'tcx>> + Captures<'a> {
+        self.head().flatten_or_pat().into_iter().map(move |pat| {
+            let mut new = self.clone();
+            new.pats[0] = pat;
+            new
+        })
+    }
+
+    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
+    /// Only call if `ctor.is_covered_by(self.head().ctor())` is true.
+    fn pop_head_constructor(
+        &self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        ctor: &Constructor<'tcx>,
+    ) -> PatStack<'p, 'tcx> {
+        // We pop the head pattern and push the new fields extracted from the arguments of
+        // `self.head()`.
+        let mut new_pats = self.head().specialize(pcx, ctor);
+        new_pats.extend_from_slice(&self.pats[1..]);
+        PatStack { pats: new_pats }
+    }
+}
+
+impl<'p, 'tcx> fmt::Debug for PatStack<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // We pretty-print similarly to the `Debug` impl of `Matrix`.
+        write!(f, "+")?;
+        for pat in self.iter() {
+            write!(f, " {pat:?} +")?;
+        }
+        Ok(())
+    }
+}
+
+/// A row of the matrix.
+#[derive(Clone)]
+struct MatrixRow<'p, 'tcx> {
+    // The patterns in the row.
+    pats: PatStack<'p, 'tcx>,
+    /// Whether the original arm had a guard. This is inherited when specializing.
+    is_under_guard: bool,
+    /// When we specialize, we remember which row of the original matrix produced a given row of the
+    /// specialized matrix. When we unspecialize, we use this to propagate usefulness back up the
+    /// callstack.
+    parent_row: usize,
+    /// False when the matrix is just built. This is set to `true` by
+    /// [`compute_exhaustiveness_and_usefulness`] if the arm is found to be useful.
+    /// This is reset to `false` when specializing.
+    useful: bool,
+}
+
+impl<'p, 'tcx> MatrixRow<'p, 'tcx> {
+    fn is_empty(&self) -> bool {
+        self.pats.is_empty()
+    }
+
+    fn len(&self) -> usize {
+        self.pats.len()
+    }
+
+    fn head(&self) -> &'p DeconstructedPat<'p, 'tcx> {
+        self.pats.head()
+    }
+
+    fn iter(&self) -> impl Iterator<Item = &DeconstructedPat<'p, 'tcx>> {
+        self.pats.iter()
+    }
+
+    // Recursively expand the first or-pattern into its subpatterns. Only useful if the pattern is
+    // an or-pattern. Panics if `self` is empty.
+    fn expand_or_pat<'a>(&'a self) -> impl Iterator<Item = MatrixRow<'p, 'tcx>> + Captures<'a> {
+        self.pats.expand_or_pat().map(|patstack| MatrixRow {
+            pats: patstack,
+            parent_row: self.parent_row,
+            is_under_guard: self.is_under_guard,
+            useful: false,
+        })
+    }
+
+    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
+    /// Only call if `ctor.is_covered_by(self.head().ctor())` is true.
+    fn pop_head_constructor(
+        &self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        ctor: &Constructor<'tcx>,
+        parent_row: usize,
+    ) -> MatrixRow<'p, 'tcx> {
+        MatrixRow {
+            pats: self.pats.pop_head_constructor(pcx, ctor),
+            parent_row,
+            is_under_guard: self.is_under_guard,
+            useful: false,
+        }
+    }
+}
+
+impl<'p, 'tcx> fmt::Debug for MatrixRow<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.pats.fmt(f)
+    }
+}
+
+/// A 2D matrix. Represents a list of pattern-tuples under investigation.
+///
+/// Invariant: each row must have the same length, and each column must have the same type.
+///
+/// Invariant: the first column must not contain or-patterns. This is handled by
+/// [`Matrix::expand_and_push`].
+///
+/// In fact each column corresponds to a place inside the scrutinee of the match. E.g. after
+/// specializing `(,)` and `Some` on a pattern of type `(Option<u32>, bool)`, the first column of
+/// the matrix will correspond to `scrutinee.0.Some.0` and the second column to `scrutinee.1`.
+#[derive(Clone)]
+struct Matrix<'p, 'tcx> {
+    /// Vector of rows. The rows must form a rectangular 2D array. Moreover, all the patterns of
+    /// each column must have the same type. Each column corresponds to a place within the
+    /// scrutinee.
+    rows: Vec<MatrixRow<'p, 'tcx>>,
+    /// Stores an extra fictitious row full of wildcards. Mostly used to keep track of the type of
+    /// each column. This must obey the same invariants as the real rows.
+    wildcard_row: PatStack<'p, 'tcx>,
+    /// Track for each column/place whether it contains a known valid value.
+    place_validity: SmallVec<[ValidityConstraint; 2]>,
+}
+
+impl<'p, 'tcx> Matrix<'p, 'tcx> {
+    /// Pushes a new row to the matrix. If the row starts with an or-pattern, this recursively
+    /// expands it. Internal method, prefer [`Matrix::new`].
+    fn expand_and_push(&mut self, row: MatrixRow<'p, 'tcx>) {
+        if !row.is_empty() && row.head().is_or_pat() {
+            // Expand nested or-patterns.
+            for new_row in row.expand_or_pat() {
+                self.rows.push(new_row);
+            }
+        } else {
+            self.rows.push(row);
+        }
+    }
+
+    /// Build a new matrix from an iterator of `MatchArm`s.
+    fn new<'a>(
+        cx: &MatchCheckCtxt<'p, 'tcx>,
+        arms: &[MatchArm<'p, 'tcx>],
+        scrut_ty: Ty<'tcx>,
+        scrut_validity: ValidityConstraint,
+    ) -> Self
+    where
+        'p: 'a,
+    {
+        let wild_pattern = cx.pattern_arena.alloc(DeconstructedPat::wildcard(scrut_ty, DUMMY_SP));
+        let wildcard_row = PatStack::from_pattern(wild_pattern);
+        let mut matrix = Matrix {
+            rows: Vec::with_capacity(arms.len()),
+            wildcard_row,
+            place_validity: smallvec![scrut_validity],
+        };
+        for (row_id, arm) in arms.iter().enumerate() {
+            let v = MatrixRow {
+                pats: PatStack::from_pattern(arm.pat),
+                parent_row: row_id, // dummy, we won't read it
+                is_under_guard: arm.has_guard,
+                useful: false,
+            };
+            matrix.expand_and_push(v);
+        }
+        matrix
+    }
+
+    fn head_ty(&self) -> Option<Ty<'tcx>> {
+        if self.column_count() == 0 {
+            return None;
+        }
+
+        let mut ty = self.wildcard_row.head().ty();
+        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
+        // version. Otherwise we could encounter constructors for the revealed type and crash.
+        let is_opaque = |ty: Ty<'tcx>| matches!(ty.kind(), ty::Alias(ty::Opaque, ..));
+        if is_opaque(ty) {
+            for pat in self.heads() {
+                let pat_ty = pat.ty();
+                if !is_opaque(pat_ty) {
+                    ty = pat_ty;
+                    break;
+                }
+            }
+        }
+        Some(ty)
+    }
+    fn column_count(&self) -> usize {
+        self.wildcard_row.len()
+    }
+
+    fn rows<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'a MatrixRow<'p, 'tcx>> + Clone + DoubleEndedIterator + ExactSizeIterator
+    {
+        self.rows.iter()
+    }
+    fn rows_mut<'a>(
+        &'a mut self,
+    ) -> impl Iterator<Item = &'a mut MatrixRow<'p, 'tcx>> + DoubleEndedIterator + ExactSizeIterator
+    {
+        self.rows.iter_mut()
+    }
+
+    /// Iterate over the first pattern of each row.
+    fn heads<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Clone + Captures<'a> {
+        self.rows().map(|r| r.head())
+    }
+
+    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
+    fn specialize_constructor(
+        &self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        ctor: &Constructor<'tcx>,
+    ) -> Matrix<'p, 'tcx> {
+        let wildcard_row = self.wildcard_row.pop_head_constructor(pcx, ctor);
+        let new_validity = self.place_validity[0].specialize(pcx, ctor);
+        let new_place_validity = std::iter::repeat(new_validity)
+            .take(ctor.arity(pcx))
+            .chain(self.place_validity[1..].iter().copied())
+            .collect();
+        let mut matrix =
+            Matrix { rows: Vec::new(), wildcard_row, place_validity: new_place_validity };
+        for (i, row) in self.rows().enumerate() {
+            if ctor.is_covered_by(pcx, row.head().ctor()) {
+                let new_row = row.pop_head_constructor(pcx, ctor, i);
+                matrix.expand_and_push(new_row);
+            }
+        }
+        matrix
+    }
+}
+
+/// Pretty-printer for matrices of patterns, example:
+///
+/// ```text
+/// + _     + []                +
+/// + true  + [First]           +
+/// + true  + [Second(true)]    +
+/// + false + [_]               +
+/// + _     + [_, _, tail @ ..] +
+/// | ✓     | ?                 | // column validity
+/// ```
+impl<'p, 'tcx> fmt::Debug for Matrix<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "\n")?;
+
+        let mut pretty_printed_matrix: Vec<Vec<String>> = self
+            .rows
+            .iter()
+            .map(|row| row.iter().map(|pat| format!("{pat:?}")).collect())
+            .collect();
+        pretty_printed_matrix
+            .push(self.place_validity.iter().map(|validity| format!("{validity}")).collect());
+
+        let column_count = self.column_count();
+        assert!(self.rows.iter().all(|row| row.len() == column_count));
+        assert!(self.place_validity.len() == column_count);
+        let column_widths: Vec<usize> = (0..column_count)
+            .map(|col| pretty_printed_matrix.iter().map(|row| row[col].len()).max().unwrap_or(0))
+            .collect();
+
+        for (row_i, row) in pretty_printed_matrix.into_iter().enumerate() {
+            let is_validity_row = row_i == self.rows.len();
+            let sep = if is_validity_row { "|" } else { "+" };
+            write!(f, "{sep}")?;
+            for (column, pat_str) in row.into_iter().enumerate() {
+                write!(f, " ")?;
+                write!(f, "{:1$}", pat_str, column_widths[column])?;
+                write!(f, " {sep}")?;
+            }
+            if is_validity_row {
+                write!(f, " // column validity")?;
+            }
+            write!(f, "\n")?;
+        }
+        Ok(())
+    }
+}
+
+/// A witness-tuple of non-exhaustiveness for error reporting, represented as a list of patterns (in
+/// reverse order of construction).
+///
+/// This mirrors `PatStack`: they function similarly, except `PatStack` contains user patterns we
+/// are inspecting, and `WitnessStack` contains witnesses we are constructing.
+/// FIXME(Nadrieril): use the same order of patterns for both.
+///
+/// A `WitnessStack` should have the same types and length as the `PatStack`s we are inspecting
+/// (except we store the patterns in reverse order). The same way `PatStack` starts with length 1,
+/// at the end of the algorithm this will have length 1. In the middle of the algorithm, it can
+/// contain multiple patterns.
+///
+/// For example, if we are constructing a witness for the match against
+///
+/// ```compile_fail,E0004
+/// struct Pair(Option<(u32, u32)>, bool);
+/// # fn foo(p: Pair) {
+/// match p {
+///    Pair(None, _) => {}
+///    Pair(_, false) => {}
+/// }
+/// # }
+/// ```
+///
+/// We'll perform the following steps (among others):
+/// ```text
+/// - Start with a matrix representing the match
+///     `PatStack(vec![Pair(None, _)])`
+///     `PatStack(vec![Pair(_, false)])`
+/// - Specialize with `Pair`
+///     `PatStack(vec![None, _])`
+///     `PatStack(vec![_, false])`
+/// - Specialize with `Some`
+///     `PatStack(vec![_, false])`
+/// - Specialize with `_`
+///     `PatStack(vec![false])`
+/// - Specialize with `true`
+///     // no patstacks left
+/// - This is a non-exhaustive match: we have the empty witness stack as a witness.
+///     `WitnessStack(vec![])`
+/// - Apply `true`
+///     `WitnessStack(vec![true])`
+/// - Apply `_`
+///     `WitnessStack(vec![true, _])`
+/// - Apply `Some`
+///     `WitnessStack(vec![true, Some(_)])`
+/// - Apply `Pair`
+///     `WitnessStack(vec![Pair(Some(_), true)])`
+/// ```
+///
+/// The final `Pair(Some(_), true)` is then the resulting witness.
+///
+/// See the top of the file for more detailed explanations and examples.
+#[derive(Debug, Clone)]
+pub(crate) struct WitnessStack<'tcx>(Vec<WitnessPat<'tcx>>);
+
+impl<'tcx> WitnessStack<'tcx> {
+    /// Asserts that the witness contains a single pattern, and returns it.
+    fn single_pattern(self) -> WitnessPat<'tcx> {
+        assert_eq!(self.0.len(), 1);
+        self.0.into_iter().next().unwrap()
+    }
+
+    /// Reverses specialization by the `Missing` constructor by pushing a whole new pattern.
+    fn push_pattern(&mut self, pat: WitnessPat<'tcx>) {
+        self.0.push(pat);
+    }
+
+    /// Reverses specialization. Given a witness obtained after specialization, this constructs a
+    /// new witness valid for before specialization. See the section on `unspecialize` at the top of
+    /// the file.
+    ///
+    /// Examples:
+    /// ```text
+    /// ctor: tuple of 2 elements
+    /// pats: [false, "foo", _, true]
+    /// result: [(false, "foo"), _, true]
+    ///
+    /// ctor: Enum::Variant { a: (bool, &'static str), b: usize}
+    /// pats: [(false, "foo"), _, true]
+    /// result: [Enum::Variant { a: (false, "foo"), b: _ }, true]
+    /// ```
+    fn apply_constructor(&mut self, pcx: &PatCtxt<'_, '_, 'tcx>, ctor: &Constructor<'tcx>) {
+        let len = self.0.len();
+        let arity = ctor.arity(pcx);
+        let fields = self.0.drain((len - arity)..).rev().collect();
+        let pat = WitnessPat::new(ctor.clone(), fields, pcx.ty);
+        self.0.push(pat);
+    }
+}
+
+/// Represents a set of pattern-tuples that are witnesses of non-exhaustiveness for error
+/// reporting. This has similar invariants as `Matrix` does.
+///
+/// The `WitnessMatrix` returned by [`compute_exhaustiveness_and_usefulness`] obeys the invariant
+/// that the union of the input `Matrix` and the output `WitnessMatrix` together matches the type
+/// exhaustively.
+///
+/// Just as the `Matrix` starts with a single column, by the end of the algorithm, this has a single
+/// column, which contains the patterns that are missing for the match to be exhaustive.
+#[derive(Debug, Clone)]
+pub struct WitnessMatrix<'tcx>(Vec<WitnessStack<'tcx>>);
+
+impl<'tcx> WitnessMatrix<'tcx> {
+    /// New matrix with no witnesses.
+    fn empty() -> Self {
+        WitnessMatrix(vec![])
+    }
+    /// New matrix with one `()` witness, i.e. with no columns.
+    fn unit_witness() -> Self {
+        WitnessMatrix(vec![WitnessStack(vec![])])
+    }
+
+    /// Whether this has any witnesses.
+    fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+    /// Asserts that there is a single column and returns the patterns in it.
+    fn single_column(self) -> Vec<WitnessPat<'tcx>> {
+        self.0.into_iter().map(|w| w.single_pattern()).collect()
+    }
+
+    /// Reverses specialization by the `Missing` constructor by pushing a whole new pattern.
+    fn push_pattern(&mut self, pat: WitnessPat<'tcx>) {
+        for witness in self.0.iter_mut() {
+            witness.push_pattern(pat.clone())
+        }
+    }
+
+    /// Reverses specialization by `ctor`. See the section on `unspecialize` at the top of the file.
+    fn apply_constructor(
+        &mut self,
+        pcx: &PatCtxt<'_, '_, 'tcx>,
+        missing_ctors: &[Constructor<'tcx>],
+        ctor: &Constructor<'tcx>,
+        report_individual_missing_ctors: bool,
+    ) {
+        if self.is_empty() {
+            return;
+        }
+        if matches!(ctor, Constructor::Missing) {
+            // We got the special `Missing` constructor that stands for the constructors not present
+            // in the match.
+            if !report_individual_missing_ctors {
+                // Report `_` as missing.
+                let pat = WitnessPat::wild_from_ctor(pcx, Constructor::Wildcard);
+                self.push_pattern(pat);
+            } else if missing_ctors.iter().any(|c| c.is_non_exhaustive()) {
+                // We need to report a `_` anyway, so listing other constructors would be redundant.
+                // `NonExhaustive` is displayed as `_` just like `Wildcard`, but it will be picked
+                // up by diagnostics to add a note about why `_` is required here.
+                let pat = WitnessPat::wild_from_ctor(pcx, Constructor::NonExhaustive);
+                self.push_pattern(pat);
+            } else {
+                // For each missing constructor `c`, we add a `c(_, _, _)` witness appropriately
+                // filled with wildcards.
+                let mut ret = Self::empty();
+                for ctor in missing_ctors {
+                    let pat = WitnessPat::wild_from_ctor(pcx, ctor.clone());
+                    // Clone `self` and add `c(_, _, _)` to each of its witnesses.
+                    let mut wit_matrix = self.clone();
+                    wit_matrix.push_pattern(pat);
+                    ret.extend(wit_matrix);
+                }
+                *self = ret;
+            }
+        } else {
+            // Any other constructor we unspecialize as expected.
+            for witness in self.0.iter_mut() {
+                witness.apply_constructor(pcx, ctor)
+            }
+        }
+    }
+
+    /// Merges the witnesses of two matrices. Their column types must match.
+    fn extend(&mut self, other: Self) {
+        self.0.extend(other.0)
+    }
+}
+
+/// The core of the algorithm.
+///
+/// This recursively computes witnesses of the non-exhaustiveness of `matrix` (if any). Also tracks
+/// usefulness of each row in the matrix (in `row.useful`). We track usefulness of each
+/// subpattern using interior mutability in `DeconstructedPat`.
+///
+/// The input `Matrix` and the output `WitnessMatrix` together match the type exhaustively.
+///
+/// The key steps are:
+/// - specialization, where we dig into the rows that have a specific constructor and call ourselves
+///     recursively;
+/// - unspecialization, where we lift the results from the previous step into results for this step
+///     (using `apply_constructor` and by updating `row.useful` for each parent row).
+/// This is all explained at the top of the file.
+#[instrument(level = "debug", skip(cx, is_top_level), ret)]
+fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    matrix: &mut Matrix<'p, 'tcx>,
+    is_top_level: bool,
+) -> WitnessMatrix<'tcx> {
+    debug_assert!(matrix.rows().all(|r| r.len() == matrix.column_count()));
+
+    let Some(ty) = matrix.head_ty() else {
+        // The base case: there are no columns in the matrix. We are morally pattern-matching on ().
+        // A row is useful iff it has no (unguarded) rows above it.
+        for row in matrix.rows_mut() {
+            // All rows are useful until they're not.
+            row.useful = true;
+            // When there's an unguarded row, the match is exhaustive and any subsequent row is not
+            // useful.
+            if !row.is_under_guard {
+                return WitnessMatrix::empty();
+            }
+        }
+        // No (unguarded) rows, so the match is not exhaustive. We return a new witness.
+        return WitnessMatrix::unit_witness();
+    };
+
+    debug!("ty: {ty:?}");
+    let pcx = &PatCtxt { cx, ty, is_top_level };
+
+    // Whether the place/column we are inspecting is known to contain valid data.
+    let place_validity = matrix.place_validity[0];
+    // For backwards compability we allow omitting some empty arms that we ideally shouldn't.
+    let place_validity = place_validity.allow_omitting_side_effecting_arms();
+
+    // Analyze the constructors present in this column.
+    let ctors = matrix.heads().map(|p| p.ctor());
+    let split_set = ConstructorSet::for_ty(cx, ty).split(pcx, ctors);
+    let all_missing = split_set.present.is_empty();
+
+    // Build the set of constructors we will specialize with. It must cover the whole type.
+    let mut split_ctors = split_set.present;
+    if !split_set.missing.is_empty() {
+        // We need to iterate over a full set of constructors, so we add `Missing` to represent the
+        // missing ones. This is explained under "Constructor Splitting" at the top of this file.
+        split_ctors.push(Constructor::Missing);
+    } else if !split_set.missing_empty.is_empty() && !place_validity.is_known_valid() {
+        // The missing empty constructors are reachable if the place can contain invalid data.
+        split_ctors.push(Constructor::Missing);
+    }
+
+    // Decide what constructors to report.
+    let always_report_all = is_top_level && !IntRange::is_integral(pcx.ty);
+    // Whether we should report "Enum::A and Enum::C are missing" or "_ is missing".
+    let report_individual_missing_ctors = always_report_all || !all_missing;
+    // Which constructors are considered missing. We ensure that `!missing_ctors.is_empty() =>
+    // split_ctors.contains(Missing)`. The converse usually holds except in the
+    // `MaybeInvalidButAllowOmittingArms` backwards-compatibility case.
+    let mut missing_ctors = split_set.missing;
+    if !place_validity.allows_omitting_empty_arms() {
+        missing_ctors.extend(split_set.missing_empty);
+    }
+
+    let mut ret = WitnessMatrix::empty();
+    for ctor in split_ctors {
+        debug!("specialize({:?})", ctor);
+        // Dig into rows that match `ctor`.
+        let mut spec_matrix = matrix.specialize_constructor(pcx, &ctor);
+        let mut witnesses = ensure_sufficient_stack(|| {
+            compute_exhaustiveness_and_usefulness(cx, &mut spec_matrix, false)
+        });
+
+        let counts_for_exhaustiveness = match ctor {
+            Constructor::Missing => !missing_ctors.is_empty(),
+            // If there are missing constructors we'll report those instead. Since `Missing` matches
+            // only the wildcard rows, it matches fewer rows than this constructor, and is therefore
+            // guaranteed to result in the same or more witnesses. So skipping this does not
+            // jeopardize correctness.
+            _ => missing_ctors.is_empty(),
+        };
+        if counts_for_exhaustiveness {
+            // Transform witnesses for `spec_matrix` into witnesses for `matrix`.
+            witnesses.apply_constructor(
+                pcx,
+                &missing_ctors,
+                &ctor,
+                report_individual_missing_ctors,
+            );
+            // Accumulate the found witnesses.
+            ret.extend(witnesses);
+        }
+
+        // A parent row is useful if any of its children is.
+        for child_row in spec_matrix.rows() {
+            let parent_row = &mut matrix.rows[child_row.parent_row];
+            parent_row.useful = parent_row.useful || child_row.useful;
+        }
+    }
+
+    // Record usefulness in the patterns.
+    for row in matrix.rows() {
+        if row.useful {
+            row.head().set_useful();
+        }
+    }
+
+    ret
+}
+
+/// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
+/// inspect the same subvalue/place".
+/// This is used to traverse patterns column-by-column for lints. Despite similarities with
+/// [`compute_exhaustiveness_and_usefulness`], this does a different traversal. Notably this is
+/// linear in the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case
+/// exponential (exhaustiveness is NP-complete). The core difference is that we treat sub-columns
+/// separately.
+///
+/// This must not contain an or-pattern. `specialize` takes care to expand them.
+///
+/// This is not used in the main algorithm; only in lints.
+#[derive(Debug)]
+struct PatternColumn<'p, 'tcx> {
+    patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>,
+}
+
+impl<'p, 'tcx> PatternColumn<'p, 'tcx> {
+    fn new(patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>) -> Self {
+        Self { patterns }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.patterns.is_empty()
+    }
+    fn head_ty(&self) -> Option<Ty<'tcx>> {
+        if self.patterns.len() == 0 {
+            return None;
+        }
+        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
+        // version. Otherwise we could encounter constructors for the revealed type and crash.
+        let is_opaque = |ty: Ty<'tcx>| matches!(ty.kind(), ty::Alias(ty::Opaque, ..));
+        let first_ty = self.patterns[0].ty();
+        if is_opaque(first_ty) {
+            for pat in &self.patterns {
+                let ty = pat.ty();
+                if !is_opaque(ty) {
+                    return Some(ty);
+                }
+            }
+        }
+        Some(first_ty)
+    }
+
+    /// Do constructor splitting on the constructors of the column.
+    fn analyze_ctors(&self, pcx: &PatCtxt<'_, 'p, 'tcx>) -> SplitConstructorSet<'tcx> {
+        let column_ctors = self.patterns.iter().map(|p| p.ctor());
+        ConstructorSet::for_ty(pcx.cx, pcx.ty).split(pcx, column_ctors)
+    }
+
+    fn iter<'a>(&'a self) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
+        self.patterns.iter().copied()
+    }
+
+    /// Does specialization: given a constructor, this takes the patterns from the column that match
+    /// the constructor, and outputs their fields.
+    /// This returns one column per field of the constructor. They usually all have the same length
+    /// (the number of patterns in `self` that matched `ctor`), except that we expand or-patterns
+    /// which may change the lengths.
+    fn specialize(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>) -> Vec<Self> {
+        let arity = ctor.arity(pcx);
+        if arity == 0 {
+            return Vec::new();
+        }
+
+        // We specialize the column by `ctor`. This gives us `arity`-many columns of patterns. These
+        // columns may have different lengths in the presence of or-patterns (this is why we can't
+        // reuse `Matrix`).
+        let mut specialized_columns: Vec<_> =
+            (0..arity).map(|_| Self { patterns: Vec::new() }).collect();
+        let relevant_patterns =
+            self.patterns.iter().filter(|pat| ctor.is_covered_by(pcx, pat.ctor()));
+        for pat in relevant_patterns {
+            let specialized = pat.specialize(pcx, ctor);
+            for (subpat, column) in specialized.iter().zip(&mut specialized_columns) {
+                if subpat.is_or_pat() {
+                    column.patterns.extend(subpat.flatten_or_pat())
+                } else {
+                    column.patterns.push(subpat)
+                }
+            }
+        }
+
+        assert!(
+            !specialized_columns[0].is_empty(),
+            "ctor {ctor:?} was listed as present but isn't;
+            there is an inconsistency between `Constructor::is_covered_by` and `ConstructorSet::split`"
+        );
+        specialized_columns
+    }
+}
+
+/// Traverse the patterns to collect any variants of a non_exhaustive enum that fail to be mentioned
+/// in a given column.
+#[instrument(level = "debug", skip(cx), ret)]
+fn collect_nonexhaustive_missing_variants<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    column: &PatternColumn<'p, 'tcx>,
+) -> Vec<WitnessPat<'tcx>> {
+    let Some(ty) = column.head_ty() else {
+        return Vec::new();
+    };
+    let pcx = &PatCtxt::new_dummy(cx, ty);
+
+    let set = column.analyze_ctors(pcx);
+    if set.present.is_empty() {
+        // We can't consistently handle the case where no constructors are present (since this would
+        // require digging deep through any type in case there's a non_exhaustive enum somewhere),
+        // so for consistency we refuse to handle the top-level case, where we could handle it.
+        return vec![];
+    }
+
+    let mut witnesses = Vec::new();
+    if cx.is_foreign_non_exhaustive_enum(ty) {
+        witnesses.extend(
+            set.missing
+                .into_iter()
+                // This will list missing visible variants.
+                .filter(|c| !matches!(c, Constructor::Hidden | Constructor::NonExhaustive))
+                .map(|missing_ctor| WitnessPat::wild_from_ctor(pcx, missing_ctor)),
+        )
+    }
+
+    // Recurse into the fields.
+    for ctor in set.present {
+        let specialized_columns = column.specialize(pcx, &ctor);
+        let wild_pat = WitnessPat::wild_from_ctor(pcx, ctor);
+        for (i, col_i) in specialized_columns.iter().enumerate() {
+            // Compute witnesses for each column.
+            let wits_for_col_i = collect_nonexhaustive_missing_variants(cx, col_i);
+            // For each witness, we build a new pattern in the shape of `ctor(_, _, wit, _, _)`,
+            // adding enough wildcards to match `arity`.
+            for wit in wits_for_col_i {
+                let mut pat = wild_pat.clone();
+                pat.fields[i] = wit;
+                witnesses.push(pat);
+            }
+        }
+    }
+    witnesses
+}
+
+/// Traverse the patterns to warn the user about ranges that overlap on their endpoints.
+#[instrument(level = "debug", skip(cx))]
+fn lint_overlapping_range_endpoints<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    column: &PatternColumn<'p, 'tcx>,
+) {
+    let Some(ty) = column.head_ty() else {
+        return;
+    };
+    let pcx = &PatCtxt::new_dummy(cx, ty);
+
+    let set = column.analyze_ctors(pcx);
+
+    if IntRange::is_integral(ty) {
+        let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
+            let overlap_as_pat = overlap.to_diagnostic_pat(ty, cx.tcx);
+            let overlaps: Vec<_> = overlapped_spans
+                .iter()
+                .copied()
+                .map(|span| Overlap { range: overlap_as_pat.clone(), span })
+                .collect();
+            cx.tcx.emit_spanned_lint(
+                lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
+                cx.match_lint_level,
+                this_span,
+                OverlappingRangeEndpoints { overlap: overlaps, range: this_span },
+            );
+        };
+
+        // If two ranges overlapped, the split set will contain their intersection as a singleton.
+        let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
+        for overlap_range in split_int_ranges.clone() {
+            if overlap_range.is_singleton() {
+                let overlap: MaybeInfiniteInt = overlap_range.lo;
+                // Ranges that look like `lo..=overlap`.
+                let mut prefixes: SmallVec<[_; 1]> = Default::default();
+                // Ranges that look like `overlap..=hi`.
+                let mut suffixes: SmallVec<[_; 1]> = Default::default();
+                // Iterate on patterns that contained `overlap`.
+                for pat in column.iter() {
+                    let this_span = pat.span();
+                    let Constructor::IntRange(this_range) = pat.ctor() else { continue };
+                    if this_range.is_singleton() {
+                        // Don't lint when one of the ranges is a singleton.
+                        continue;
+                    }
+                    if this_range.lo == overlap {
+                        // `this_range` looks like `overlap..=this_range.hi`; it overlaps with any
+                        // ranges that look like `lo..=overlap`.
+                        if !prefixes.is_empty() {
+                            emit_lint(overlap_range, this_span, &prefixes);
+                        }
+                        suffixes.push(this_span)
+                    } else if this_range.hi == overlap.plus_one() {
+                        // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
+                        // ranges that look like `overlap..=hi`.
+                        if !suffixes.is_empty() {
+                            emit_lint(overlap_range, this_span, &suffixes);
+                        }
+                        prefixes.push(this_span)
+                    }
+                }
+            }
+        }
+    } else {
+        // Recurse into the fields.
+        for ctor in set.present {
+            for col in column.specialize(pcx, &ctor) {
+                lint_overlapping_range_endpoints(cx, &col);
+            }
+        }
+    }
+}
+
+/// The arm of a match expression.
+#[derive(Clone, Copy, Debug)]
+pub struct MatchArm<'p, 'tcx> {
+    /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`.
+    pub pat: &'p DeconstructedPat<'p, 'tcx>,
+    pub hir_id: HirId,
+    pub has_guard: bool,
+}
+
+/// Indicates whether or not a given arm is useful.
+#[derive(Clone, Debug)]
+pub enum Usefulness {
+    /// The arm is useful. This additionally carries a set of or-pattern branches that have been
+    /// found to be redundant despite the overall arm being useful. Used only in the presence of
+    /// or-patterns, otherwise it stays empty.
+    Useful(Vec<Span>),
+    /// The arm is redundant and can be removed without changing the behavior of the match
+    /// expression.
+    Redundant,
+}
+
+/// The output of checking a match for exhaustiveness and arm usefulness.
+pub struct UsefulnessReport<'p, 'tcx> {
+    /// For each arm of the input, whether that arm is useful after the arms above it.
+    pub arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Usefulness)>,
+    /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of
+    /// exhaustiveness.
+    pub non_exhaustiveness_witnesses: Vec<WitnessPat<'tcx>>,
+}
+
+/// The entrypoint for this file. Computes whether a match is exhaustive and which of its arms are
+/// useful.
+#[instrument(skip(cx, arms), level = "debug")]
+pub fn compute_match_usefulness<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    arms: &[MatchArm<'p, 'tcx>],
+    scrut_ty: Ty<'tcx>,
+) -> UsefulnessReport<'p, 'tcx> {
+    let scrut_validity = ValidityConstraint::from_bool(cx.known_valid_scrutinee);
+    let mut matrix = Matrix::new(cx, arms, scrut_ty, scrut_validity);
+    let non_exhaustiveness_witnesses = compute_exhaustiveness_and_usefulness(cx, &mut matrix, true);
+
+    let non_exhaustiveness_witnesses: Vec<_> = non_exhaustiveness_witnesses.single_column();
+    let arm_usefulness: Vec<_> = arms
+        .iter()
+        .copied()
+        .map(|arm| {
+            debug!(?arm);
+            // We warn when a pattern is not useful.
+            let usefulness = if arm.pat.is_useful() {
+                Usefulness::Useful(arm.pat.redundant_spans())
+            } else {
+                Usefulness::Redundant
+            };
+            (arm, usefulness)
+        })
+        .collect();
+    let report = UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses };
+
+    let pat_column = PatternColumn::new(matrix.heads().collect());
+    // Lint on ranges that overlap on their endpoints, which is likely a mistake.
+    lint_overlapping_range_endpoints(cx, &pat_column);
+
+    // Run the non_exhaustive_omitted_patterns lint. Only run on refutable patterns to avoid hitting
+    // `if let`s. Only run if the match is exhaustive otherwise the error is redundant.
+    if cx.refutable && report.non_exhaustiveness_witnesses.is_empty() {
+        if !matches!(
+            cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, cx.match_lint_level).0,
+            rustc_session::lint::Level::Allow
+        ) {
+            let witnesses = collect_nonexhaustive_missing_variants(cx, &pat_column);
+            if !witnesses.is_empty() {
+                // Report that a match of a `non_exhaustive` enum marked with `non_exhaustive_omitted_patterns`
+                // is not exhaustive enough.
+                //
+                // NB: The partner lint for structs lives in `compiler/rustc_hir_analysis/src/check/pat.rs`.
+                cx.tcx.emit_spanned_lint(
+                    NON_EXHAUSTIVE_OMITTED_PATTERNS,
+                    cx.match_lint_level,
+                    cx.scrut_span,
+                    NonExhaustiveOmittedPattern {
+                        scrut_ty,
+                        uncovered: Uncovered::new(cx.scrut_span, cx, witnesses),
+                    },
+                );
+            }
+        } else {
+            // We used to allow putting the `#[allow(non_exhaustive_omitted_patterns)]` on a match
+            // arm. This no longer makes sense so we warn users, to avoid silently breaking their
+            // usage of the lint.
+            for arm in arms {
+                let (lint_level, lint_level_source) =
+                    cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, arm.hir_id);
+                if !matches!(lint_level, rustc_session::lint::Level::Allow) {
+                    let decorator = NonExhaustiveOmittedPatternLintOnArm {
+                        lint_span: lint_level_source.span(),
+                        suggest_lint_on_match: cx.whole_match_span.map(|span| span.shrink_to_lo()),
+                        lint_level: lint_level.as_str(),
+                        lint_name: "non_exhaustive_omitted_patterns",
+                    };
+
+                    use rustc_errors::DecorateLint;
+                    let mut err = cx.tcx.sess.struct_span_warn(arm.pat.span(), "");
+                    err.set_primary_message(decorator.msg());
+                    decorator.decorate_lint(&mut err);
+                    err.emit();
+                }
+            }
+        }
+    }
+
+    report
+}
-- 
cgit 1.4.1-3-g733a5


From 3691a0aee58051298df67c897582ade1e69df95b Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sun, 10 Dec 2023 22:14:00 +0100
Subject: Gather rustc-specific functions around `MatchCheckCtxt`

---
 compiler/rustc_mir_build/src/errors.rs             |   2 +-
 .../src/thir/pattern/check_match.rs                |  21 +-
 compiler/rustc_pattern_analysis/src/constructor.rs | 278 +------
 compiler/rustc_pattern_analysis/src/cx.rs          | 837 +++++++++++++++++++++
 compiler/rustc_pattern_analysis/src/errors.rs      |   8 +-
 compiler/rustc_pattern_analysis/src/lib.rs         |   1 +
 compiler/rustc_pattern_analysis/src/pat.rs         | 581 +-------------
 compiler/rustc_pattern_analysis/src/usefulness.rs  |  75 +-
 8 files changed, 903 insertions(+), 900 deletions(-)
 create mode 100644 compiler/rustc_pattern_analysis/src/cx.rs

diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 1509e2517da..9baae706dff 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -6,7 +6,7 @@ use rustc_errors::{
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::ty::{self, Ty};
-use rustc_pattern_analysis::{errors::Uncovered, usefulness::MatchCheckCtxt};
+use rustc_pattern_analysis::{cx::MatchCheckCtxt, errors::Uncovered};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index ca5823a860e..02becbead66 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -1,8 +1,9 @@
 use rustc_pattern_analysis::constructor::Constructor;
+use rustc_pattern_analysis::cx::MatchCheckCtxt;
 use rustc_pattern_analysis::errors::Uncovered;
 use rustc_pattern_analysis::pat::{DeconstructedPat, WitnessPat};
 use rustc_pattern_analysis::usefulness::{
-    compute_match_usefulness, MatchArm, MatchCheckCtxt, Usefulness, UsefulnessReport,
+    compute_match_usefulness, MatchArm, Usefulness, UsefulnessReport,
 };
 
 use crate::errors::*;
@@ -286,7 +287,7 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
                 check_borrow_conflicts_in_at_patterns(self, pat);
                 check_for_bindings_named_same_as_variants(self, pat, refutable);
             });
-            Ok(cx.pattern_arena.alloc(DeconstructedPat::from_pat(cx, pat)))
+            Ok(cx.pattern_arena.alloc(cx.lower_pat(pat)))
         }
     }
 
@@ -926,7 +927,7 @@ fn report_non_exhaustive_match<'p, 'tcx>(
         pattern = if witnesses.len() < 4 {
             witnesses
                 .iter()
-                .map(|witness| witness.to_diagnostic_pat(cx).to_string())
+                .map(|witness| cx.hoist_witness_pat(witness).to_string())
                 .collect::<Vec<String>>()
                 .join(" | ")
         } else {
@@ -950,7 +951,7 @@ fn report_non_exhaustive_match<'p, 'tcx>(
     if !is_empty_match {
         let mut non_exhaustive_tys = FxHashSet::default();
         // Look at the first witness.
-        collect_non_exhaustive_tys(cx.tcx, &witnesses[0], &mut non_exhaustive_tys);
+        collect_non_exhaustive_tys(cx, &witnesses[0], &mut non_exhaustive_tys);
 
         for ty in non_exhaustive_tys {
             if ty.is_ptr_sized_integral() {
@@ -1085,13 +1086,13 @@ fn joined_uncovered_patterns<'p, 'tcx>(
     witnesses: &[WitnessPat<'tcx>],
 ) -> String {
     const LIMIT: usize = 3;
-    let pat_to_str = |pat: &WitnessPat<'tcx>| pat.to_diagnostic_pat(cx).to_string();
+    let pat_to_str = |pat: &WitnessPat<'tcx>| cx.hoist_witness_pat(pat).to_string();
     match witnesses {
         [] => bug!(),
-        [witness] => format!("`{}`", witness.to_diagnostic_pat(cx)),
+        [witness] => format!("`{}`", cx.hoist_witness_pat(witness)),
         [head @ .., tail] if head.len() < LIMIT => {
             let head: Vec<_> = head.iter().map(pat_to_str).collect();
-            format!("`{}` and `{}`", head.join("`, `"), tail.to_diagnostic_pat(cx))
+            format!("`{}` and `{}`", head.join("`, `"), cx.hoist_witness_pat(tail))
         }
         _ => {
             let (head, tail) = witnesses.split_at(LIMIT);
@@ -1102,7 +1103,7 @@ fn joined_uncovered_patterns<'p, 'tcx>(
 }
 
 fn collect_non_exhaustive_tys<'tcx>(
-    tcx: TyCtxt<'tcx>,
+    cx: &MatchCheckCtxt<'_, 'tcx>,
     pat: &WitnessPat<'tcx>,
     non_exhaustive_tys: &mut FxHashSet<Ty<'tcx>>,
 ) {
@@ -1110,13 +1111,13 @@ fn collect_non_exhaustive_tys<'tcx>(
         non_exhaustive_tys.insert(pat.ty());
     }
     if let Constructor::IntRange(range) = pat.ctor() {
-        if range.is_beyond_boundaries(pat.ty(), tcx) {
+        if cx.is_range_beyond_boundaries(range, pat.ty()) {
             // The range denotes the values before `isize::MIN` or the values after `usize::MAX`/`isize::MAX`.
             non_exhaustive_tys.insert(pat.ty());
         }
     }
     pat.iter_fields()
-        .for_each(|field_pat| collect_non_exhaustive_tys(tcx, field_pat, non_exhaustive_tys))
+        .for_each(|field_pat| collect_non_exhaustive_tys(cx, field_pat, non_exhaustive_tys))
 }
 
 fn report_adt_defined_here<'tcx>(
diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index 4c12cb3b029..aa02ca27add 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -158,21 +158,16 @@ use rustc_apfloat::ieee::{DoubleS, IeeeFloat, SingleS};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::RangeEnd;
 use rustc_index::IndexVec;
-use rustc_middle::middle::stability::EvalResult;
-use rustc_middle::mir;
-use rustc_middle::mir::interpret::Scalar;
-use rustc_middle::thir::{Pat, PatKind, PatRange, PatRangeBoundary};
+use rustc_middle::mir::Const;
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::DUMMY_SP;
-use rustc_target::abi::{Integer, VariantIdx, FIRST_VARIANT};
+use rustc_target::abi::{Integer, VariantIdx};
 
 use self::Constructor::*;
 use self::MaybeInfiniteInt::*;
 use self::SliceKind::*;
 
-use crate::pat::Fields;
-use crate::usefulness::{MatchCheckCtxt, PatCtxt};
+use crate::usefulness::PatCtxt;
 
 /// Whether we have seen a constructor in the column or not.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
@@ -196,7 +191,7 @@ pub enum MaybeInfiniteInt {
 
 impl MaybeInfiniteInt {
     // The return value of `signed_bias` should be XORed with a value to encode/decode it.
-    fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
+    pub(crate) fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
         match *ty.kind() {
             ty::Int(ity) => {
                 let bits = Integer::from_int_ty(&tcx, ity).size().bits() as u128;
@@ -206,58 +201,13 @@ impl MaybeInfiniteInt {
         }
     }
 
-    fn new_finite(tcx: TyCtxt<'_>, ty: Ty<'_>, bits: u128) -> Self {
+    pub fn new_finite(tcx: TyCtxt<'_>, ty: Ty<'_>, bits: u128) -> Self {
         let bias = Self::signed_bias(tcx, ty);
         // Perform a shift if the underlying types are signed, which makes the interval arithmetic
         // type-independent.
         let x = bits ^ bias;
         Finite(x)
     }
-    pub(crate) fn from_pat_range_bdy<'tcx>(
-        bdy: PatRangeBoundary<'tcx>,
-        ty: Ty<'tcx>,
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> Self {
-        match bdy {
-            PatRangeBoundary::NegInfinity => NegInfinity,
-            PatRangeBoundary::Finite(value) => {
-                let bits = value.eval_bits(tcx, param_env);
-                Self::new_finite(tcx, ty, bits)
-            }
-            PatRangeBoundary::PosInfinity => PosInfinity,
-        }
-    }
-
-    /// Used only for diagnostics.
-    /// Note: it is possible to get `isize/usize::MAX+1` here, as explained in the doc for
-    /// [`IntRange::split`]. This cannot be represented as a `Const`, so we represent it with
-    /// `PosInfinity`.
-    fn to_diagnostic_pat_range_bdy<'tcx>(
-        self,
-        ty: Ty<'tcx>,
-        tcx: TyCtxt<'tcx>,
-    ) -> PatRangeBoundary<'tcx> {
-        match self {
-            NegInfinity => PatRangeBoundary::NegInfinity,
-            Finite(x) => {
-                let bias = Self::signed_bias(tcx, ty);
-                let bits = x ^ bias;
-                let size = ty.primitive_size(tcx);
-                match Scalar::try_from_uint(bits, size) {
-                    Some(scalar) => {
-                        let value = mir::Const::from_scalar(tcx, scalar, ty);
-                        PatRangeBoundary::Finite(value)
-                    }
-                    // The value doesn't fit. Since `x >= 0` and 0 always encodes the minimum value
-                    // for a type, the problem isn't that the value is too small. So it must be too
-                    // large.
-                    None => PatRangeBoundary::PosInfinity,
-                }
-            }
-            JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
-        }
-    }
 
     /// Note: this will not turn a finite value into an infinite one or vice-versa.
     pub fn minus_one(self) -> Self {
@@ -290,16 +240,11 @@ impl MaybeInfiniteInt {
 /// space: i.e., `range.lo < range.hi`.
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub struct IntRange {
-    pub(crate) lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
-    pub(crate) hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
+    pub lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
+    pub hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
 }
 
 impl IntRange {
-    #[inline]
-    pub(super) fn is_integral(ty: Ty<'_>) -> bool {
-        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_))
-    }
-
     /// Best effort; will not know that e.g. `255u8..` is a singleton.
     pub fn is_singleton(&self) -> bool {
         // Since `lo` and `hi` can't be the same `Infinity` and `plus_one` never changes from finite
@@ -421,55 +366,6 @@ impl IntRange {
                 (presence, range)
             })
     }
-
-    /// Whether the range denotes the fictitious values before `isize::MIN` or after
-    /// `usize::MAX`/`isize::MAX` (see doc of [`IntRange::split`] for why these exist).
-    pub fn is_beyond_boundaries<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool {
-        ty.is_ptr_sized_integral() && {
-            // The two invalid ranges are `NegInfinity..isize::MIN` (represented as
-            // `NegInfinity..0`), and `{u,i}size::MAX+1..PosInfinity`. `to_diagnostic_pat_range_bdy`
-            // converts `MAX+1` to `PosInfinity`, and we couldn't have `PosInfinity` in `self.lo`
-            // otherwise.
-            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            matches!(lo, PatRangeBoundary::PosInfinity)
-                || matches!(self.hi, MaybeInfiniteInt::Finite(0))
-        }
-    }
-    /// Only used for displaying the range.
-    pub(super) fn to_diagnostic_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
-        let kind = if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
-            PatKind::Wild
-        } else if self.is_singleton() {
-            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            let value = lo.as_finite().unwrap();
-            PatKind::Constant { value }
-        } else {
-            // We convert to an inclusive range for diagnostics.
-            let mut end = RangeEnd::Included;
-            let mut lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            if matches!(lo, PatRangeBoundary::PosInfinity) {
-                // The only reason to get `PosInfinity` here is the special case where
-                // `to_diagnostic_pat_range_bdy` found `{u,i}size::MAX+1`. So the range denotes the
-                // fictitious values after `{u,i}size::MAX` (see [`IntRange::split`] for why we do
-                // this). We show this to the user as `usize::MAX..` which is slightly incorrect but
-                // probably clear enough.
-                let c = ty.numeric_max_val(tcx).unwrap();
-                let value = mir::Const::from_ty_const(c, tcx);
-                lo = PatRangeBoundary::Finite(value);
-            }
-            let hi = if matches!(self.hi, MaybeInfiniteInt::Finite(0)) {
-                // The range encodes `..ty::MIN`, so we can't convert it to an inclusive range.
-                end = RangeEnd::Excluded;
-                self.hi
-            } else {
-                self.hi.minus_one()
-            };
-            let hi = hi.to_diagnostic_pat_range_bdy(ty, tcx);
-            PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
-        };
-
-        Pat { ty, span: DUMMY_SP, kind }
-    }
 }
 
 /// Note: this will render signed ranges incorrectly. To render properly, convert to a pattern
@@ -742,7 +638,7 @@ pub enum Constructor<'tcx> {
     F32Range(IeeeFloat<SingleS>, IeeeFloat<SingleS>, RangeEnd),
     F64Range(IeeeFloat<DoubleS>, IeeeFloat<DoubleS>, RangeEnd),
     /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately.
-    Str(mir::Const<'tcx>),
+    Str(Const<'tcx>),
     /// Array and slice patterns.
     Slice(Slice),
     /// Constants that must not be matched structurally. They are treated as black boxes for the
@@ -797,49 +693,10 @@ impl<'tcx> Constructor<'tcx> {
         }
     }
 
-    pub(crate) fn variant_index_for_adt(&self, adt: ty::AdtDef<'tcx>) -> VariantIdx {
-        match *self {
-            Variant(idx) => idx,
-            Single => {
-                assert!(!adt.is_enum());
-                FIRST_VARIANT
-            }
-            _ => bug!("bad constructor {:?} for adt {:?}", self, adt),
-        }
-    }
-
     /// The number of fields for this constructor. This must be kept in sync with
     /// `Fields::wildcards`.
     pub(crate) fn arity(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> usize {
-        match self {
-            Single | Variant(_) => match pcx.ty.kind() {
-                ty::Tuple(fs) => fs.len(),
-                ty::Ref(..) => 1,
-                ty::Adt(adt, ..) => {
-                    if adt.is_box() {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        1
-                    } else {
-                        let variant = &adt.variant(self.variant_index_for_adt(*adt));
-                        Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant).count()
-                    }
-                }
-                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx.ty),
-            },
-            Slice(slice) => slice.arity(),
-            Bool(..)
-            | IntRange(..)
-            | F32Range(..)
-            | F64Range(..)
-            | Str(..)
-            | Opaque(..)
-            | NonExhaustive
-            | Hidden
-            | Missing { .. }
-            | Wildcard => 0,
-            Or => bug!("The `Or` constructor doesn't have a fixed arity"),
-        }
+        pcx.cx.ctor_arity(self, pcx.ty)
     }
 
     /// Returns whether `self` is covered by `other`, i.e. whether `self` is a subset of `other`.
@@ -974,123 +831,6 @@ pub(super) struct SplitConstructorSet<'tcx> {
 }
 
 impl ConstructorSet {
-    /// Creates a set that represents all the constructors of `ty`.
-    ///
-    /// See at the top of the file for considerations of emptiness.
-    #[instrument(level = "debug", skip(cx), ret)]
-    pub fn for_ty<'p, 'tcx>(cx: &MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
-        let make_range = |start, end| {
-            IntRange::from_range(
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, start),
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, end),
-                RangeEnd::Included,
-            )
-        };
-        // This determines the set of all possible constructors for the type `ty`. For numbers,
-        // arrays and slices we use ranges and variable-length slices when appropriate.
-        match ty.kind() {
-            ty::Bool => Self::Bool,
-            ty::Char => {
-                // The valid Unicode Scalar Value ranges.
-                Self::Integers {
-                    range_1: make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
-                    range_2: Some(make_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
-                }
-            }
-            &ty::Int(ity) => {
-                let range = if ty.is_ptr_sized_integral() {
-                    // The min/max values of `isize` are not allowed to be observed.
-                    IntRange { lo: NegInfinity, hi: PosInfinity }
-                } else {
-                    let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
-                    let min = 1u128 << (bits - 1);
-                    let max = min - 1;
-                    make_range(min, max)
-                };
-                Self::Integers { range_1: range, range_2: None }
-            }
-            &ty::Uint(uty) => {
-                let range = if ty.is_ptr_sized_integral() {
-                    // The max value of `usize` is not allowed to be observed.
-                    let lo = MaybeInfiniteInt::new_finite(cx.tcx, ty, 0);
-                    IntRange { lo, hi: PosInfinity }
-                } else {
-                    let size = Integer::from_uint_ty(&cx.tcx, uty).size();
-                    let max = size.truncate(u128::MAX);
-                    make_range(0, max)
-                };
-                Self::Integers { range_1: range, range_2: None }
-            }
-            ty::Slice(sub_ty) => {
-                Self::Slice { array_len: None, subtype_is_empty: cx.is_uninhabited(*sub_ty) }
-            }
-            ty::Array(sub_ty, len) => {
-                // We treat arrays of a constant but unknown length like slices.
-                Self::Slice {
-                    array_len: len.try_eval_target_usize(cx.tcx, cx.param_env).map(|l| l as usize),
-                    subtype_is_empty: cx.is_uninhabited(*sub_ty),
-                }
-            }
-            ty::Adt(def, args) if def.is_enum() => {
-                let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(ty);
-                if def.variants().is_empty() && !is_declared_nonexhaustive {
-                    Self::NoConstructors
-                } else {
-                    let mut variants =
-                        IndexVec::from_elem(VariantVisibility::Visible, def.variants());
-                    for (idx, v) in def.variants().iter_enumerated() {
-                        let variant_def_id = def.variant(idx).def_id;
-                        // Visibly uninhabited variants.
-                        let is_inhabited = v
-                            .inhabited_predicate(cx.tcx, *def)
-                            .instantiate(cx.tcx, args)
-                            .apply(cx.tcx, cx.param_env, cx.module);
-                        // Variants that depend on a disabled unstable feature.
-                        let is_unstable = matches!(
-                            cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
-                            EvalResult::Deny { .. }
-                        );
-                        // Foreign `#[doc(hidden)]` variants.
-                        let is_doc_hidden =
-                            cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
-                        let visibility = if !is_inhabited {
-                            // FIXME: handle empty+hidden
-                            VariantVisibility::Empty
-                        } else if is_unstable || is_doc_hidden {
-                            VariantVisibility::Hidden
-                        } else {
-                            VariantVisibility::Visible
-                        };
-                        variants[idx] = visibility;
-                    }
-
-                    Self::Variants { variants, non_exhaustive: is_declared_nonexhaustive }
-                }
-            }
-            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => {
-                Self::Single { empty: cx.is_uninhabited(ty) }
-            }
-            ty::Never => Self::NoConstructors,
-            // This type is one for which we cannot list constructors, like `str` or `f64`.
-            // FIXME(Nadrieril): which of these are actually allowed?
-            ty::Float(_)
-            | ty::Str
-            | ty::Foreign(_)
-            | ty::RawPtr(_)
-            | ty::FnDef(_, _)
-            | ty::FnPtr(_)
-            | ty::Dynamic(_, _, _)
-            | ty::Closure(_, _)
-            | ty::Coroutine(_, _, _)
-            | ty::Alias(_, _)
-            | ty::Param(_)
-            | ty::Error(_) => Self::Unlistable,
-            ty::CoroutineWitness(_, _) | ty::Bound(_, _) | ty::Placeholder(_) | ty::Infer(_) => {
-                bug!("Encountered unexpected type in `ConstructorSet::for_ty`: {ty:?}")
-            }
-        }
-    }
-
     /// This analyzes a column of constructors to 1/ determine which constructors of the type (if
     /// any) are missing; 2/ split constructors to handle non-trivial intersections e.g. on ranges
     /// or slices. This can get subtle; see [`SplitConstructorSet`] for details of this operation
diff --git a/compiler/rustc_pattern_analysis/src/cx.rs b/compiler/rustc_pattern_analysis/src/cx.rs
new file mode 100644
index 00000000000..ffcbd2f5d83
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/cx.rs
@@ -0,0 +1,837 @@
+use std::fmt;
+use std::iter::once;
+
+use rustc_arena::TypedArena;
+use rustc_data_structures::captures::Captures;
+use rustc_hir::def_id::DefId;
+use rustc_hir::{HirId, RangeEnd};
+use rustc_index::Idx;
+use rustc_index::IndexVec;
+use rustc_middle::middle::stability::EvalResult;
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::Scalar;
+use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange, PatRangeBoundary};
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
+use rustc_span::{Span, DUMMY_SP};
+use rustc_target::abi::{FieldIdx, Integer, VariantIdx, FIRST_VARIANT};
+use smallvec::SmallVec;
+
+use crate::constructor::{
+    Constructor, ConstructorSet, IntRange, MaybeInfiniteInt, OpaqueId, Slice, SliceKind,
+    VariantVisibility,
+};
+use crate::pat::{DeconstructedPat, WitnessPat};
+
+use Constructor::*;
+
+pub struct MatchCheckCtxt<'p, 'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    /// The module in which the match occurs. This is necessary for
+    /// checking inhabited-ness of types because whether a type is (visibly)
+    /// inhabited can depend on whether it was defined in the current module or
+    /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
+    /// outside its module and should not be matchable with an empty match statement.
+    pub module: DefId,
+    pub param_env: ty::ParamEnv<'tcx>,
+    pub pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+    /// Lint level at the match.
+    pub match_lint_level: HirId,
+    /// The span of the whole match, if applicable.
+    pub whole_match_span: Option<Span>,
+    /// Span of the scrutinee.
+    pub scrut_span: Span,
+    /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
+    pub refutable: bool,
+    /// Whether the data at the scrutinee is known to be valid. This is false if the scrutinee comes
+    /// from a union field, a pointer deref, or a reference deref (pending opsem decisions).
+    pub known_valid_scrutinee: bool,
+}
+
+impl<'p, 'tcx> MatchCheckCtxt<'p, 'tcx> {
+    pub(super) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
+        !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
+    }
+
+    /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
+    pub fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
+        match ty.kind() {
+            ty::Adt(def, ..) => {
+                def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did().is_local()
+            }
+            _ => false,
+        }
+    }
+
+    pub(crate) fn alloc_wildcard_slice(
+        &self,
+        tys: impl IntoIterator<Item = Ty<'tcx>>,
+    ) -> &'p [DeconstructedPat<'p, 'tcx>] {
+        self.pattern_arena
+            .alloc_from_iter(tys.into_iter().map(|ty| DeconstructedPat::wildcard(ty, DUMMY_SP)))
+    }
+
+    // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
+    // uninhabited fields in order not to reveal the uninhabitedness of the whole variant.
+    // This lists the fields we keep along with their types.
+    pub(crate) fn list_variant_nonhidden_fields<'a>(
+        &'a self,
+        ty: Ty<'tcx>,
+        variant: &'a VariantDef,
+    ) -> impl Iterator<Item = (FieldIdx, Ty<'tcx>)> + Captures<'p> + Captures<'a> {
+        let cx = self;
+        let ty::Adt(adt, args) = ty.kind() else { bug!() };
+        // Whether we must not match the fields of this variant exhaustively.
+        let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
+
+        variant.fields.iter().enumerate().filter_map(move |(i, field)| {
+            let ty = field.ty(cx.tcx, args);
+            // `field.ty()` doesn't normalize after substituting.
+            let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
+            let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
+            let is_uninhabited = cx.tcx.features().exhaustive_patterns && cx.is_uninhabited(ty);
+
+            if is_uninhabited && (!is_visible || is_non_exhaustive) {
+                None
+            } else {
+                Some((FieldIdx::new(i), ty))
+            }
+        })
+    }
+
+    pub(crate) fn variant_index_for_adt(
+        ctor: &Constructor<'tcx>,
+        adt: ty::AdtDef<'tcx>,
+    ) -> VariantIdx {
+        match *ctor {
+            Variant(idx) => idx,
+            Single => {
+                assert!(!adt.is_enum());
+                FIRST_VARIANT
+            }
+            _ => bug!("bad constructor {:?} for adt {:?}", ctor, adt),
+        }
+    }
+
+    /// Creates a new list of wildcard fields for a given constructor. The result must have a length
+    /// of `ctor.arity()`.
+    #[instrument(level = "trace", skip(self))]
+    pub(crate) fn ctor_wildcard_fields(
+        &self,
+        ctor: &Constructor<'tcx>,
+        ty: Ty<'tcx>,
+    ) -> &'p [DeconstructedPat<'p, 'tcx>] {
+        let cx = self;
+        match ctor {
+            Single | Variant(_) => match ty.kind() {
+                ty::Tuple(fs) => cx.alloc_wildcard_slice(fs.iter()),
+                ty::Ref(_, rty, _) => cx.alloc_wildcard_slice(once(*rty)),
+                ty::Adt(adt, args) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        cx.alloc_wildcard_slice(once(args.type_at(0)))
+                    } else {
+                        let variant =
+                            &adt.variant(MatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
+                        let tys = cx.list_variant_nonhidden_fields(ty, variant).map(|(_, ty)| ty);
+                        cx.alloc_wildcard_slice(tys)
+                    }
+                }
+                _ => bug!("Unexpected type for `Single` constructor: {:?}", ty),
+            },
+            Slice(slice) => match *ty.kind() {
+                ty::Slice(ty) | ty::Array(ty, _) => {
+                    let arity = slice.arity();
+                    cx.alloc_wildcard_slice((0..arity).map(|_| ty))
+                }
+                _ => bug!("bad slice pattern {:?} {:?}", ctor, ty),
+            },
+            Bool(..)
+            | IntRange(..)
+            | F32Range(..)
+            | F64Range(..)
+            | Str(..)
+            | Opaque(..)
+            | NonExhaustive
+            | Hidden
+            | Missing { .. }
+            | Wildcard => &[],
+            Or => {
+                bug!("called `Fields::wildcards` on an `Or` ctor")
+            }
+        }
+    }
+
+    /// The number of fields for this constructor. This must be kept in sync with
+    /// `Fields::wildcards`.
+    pub(crate) fn ctor_arity(&self, ctor: &Constructor<'tcx>, ty: Ty<'tcx>) -> usize {
+        match ctor {
+            Single | Variant(_) => match ty.kind() {
+                ty::Tuple(fs) => fs.len(),
+                ty::Ref(..) => 1,
+                ty::Adt(adt, ..) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        1
+                    } else {
+                        let variant =
+                            &adt.variant(MatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
+                        self.list_variant_nonhidden_fields(ty, variant).count()
+                    }
+                }
+                _ => bug!("Unexpected type for `Single` constructor: {:?}", ty),
+            },
+            Slice(slice) => slice.arity(),
+            Bool(..)
+            | IntRange(..)
+            | F32Range(..)
+            | F64Range(..)
+            | Str(..)
+            | Opaque(..)
+            | NonExhaustive
+            | Hidden
+            | Missing { .. }
+            | Wildcard => 0,
+            Or => bug!("The `Or` constructor doesn't have a fixed arity"),
+        }
+    }
+
+    /// Creates a set that represents all the constructors of `ty`.
+    ///
+    /// See [`crate::constructor`] for considerations of emptiness.
+    #[instrument(level = "debug", skip(self), ret)]
+    pub fn ctors_for_ty(&self, ty: Ty<'tcx>) -> ConstructorSet {
+        let cx = self;
+        let make_range = |start, end| {
+            IntRange::from_range(
+                MaybeInfiniteInt::new_finite(cx.tcx, ty, start),
+                MaybeInfiniteInt::new_finite(cx.tcx, ty, end),
+                RangeEnd::Included,
+            )
+        };
+        // This determines the set of all possible constructors for the type `ty`. For numbers,
+        // arrays and slices we use ranges and variable-length slices when appropriate.
+        match ty.kind() {
+            ty::Bool => ConstructorSet::Bool,
+            ty::Char => {
+                // The valid Unicode Scalar Value ranges.
+                ConstructorSet::Integers {
+                    range_1: make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
+                    range_2: Some(make_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
+                }
+            }
+            &ty::Int(ity) => {
+                let range = if ty.is_ptr_sized_integral() {
+                    // The min/max values of `isize` are not allowed to be observed.
+                    IntRange {
+                        lo: MaybeInfiniteInt::NegInfinity,
+                        hi: MaybeInfiniteInt::PosInfinity,
+                    }
+                } else {
+                    let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
+                    let min = 1u128 << (bits - 1);
+                    let max = min - 1;
+                    make_range(min, max)
+                };
+                ConstructorSet::Integers { range_1: range, range_2: None }
+            }
+            &ty::Uint(uty) => {
+                let range = if ty.is_ptr_sized_integral() {
+                    // The max value of `usize` is not allowed to be observed.
+                    let lo = MaybeInfiniteInt::new_finite(cx.tcx, ty, 0);
+                    IntRange { lo, hi: MaybeInfiniteInt::PosInfinity }
+                } else {
+                    let size = Integer::from_uint_ty(&cx.tcx, uty).size();
+                    let max = size.truncate(u128::MAX);
+                    make_range(0, max)
+                };
+                ConstructorSet::Integers { range_1: range, range_2: None }
+            }
+            ty::Slice(sub_ty) => ConstructorSet::Slice {
+                array_len: None,
+                subtype_is_empty: cx.is_uninhabited(*sub_ty),
+            },
+            ty::Array(sub_ty, len) => {
+                // We treat arrays of a constant but unknown length like slices.
+                ConstructorSet::Slice {
+                    array_len: len.try_eval_target_usize(cx.tcx, cx.param_env).map(|l| l as usize),
+                    subtype_is_empty: cx.is_uninhabited(*sub_ty),
+                }
+            }
+            ty::Adt(def, args) if def.is_enum() => {
+                let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(ty);
+                if def.variants().is_empty() && !is_declared_nonexhaustive {
+                    ConstructorSet::NoConstructors
+                } else {
+                    let mut variants =
+                        IndexVec::from_elem(VariantVisibility::Visible, def.variants());
+                    for (idx, v) in def.variants().iter_enumerated() {
+                        let variant_def_id = def.variant(idx).def_id;
+                        // Visibly uninhabited variants.
+                        let is_inhabited = v
+                            .inhabited_predicate(cx.tcx, *def)
+                            .instantiate(cx.tcx, args)
+                            .apply(cx.tcx, cx.param_env, cx.module);
+                        // Variants that depend on a disabled unstable feature.
+                        let is_unstable = matches!(
+                            cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
+                            EvalResult::Deny { .. }
+                        );
+                        // Foreign `#[doc(hidden)]` variants.
+                        let is_doc_hidden =
+                            cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
+                        let visibility = if !is_inhabited {
+                            // FIXME: handle empty+hidden
+                            VariantVisibility::Empty
+                        } else if is_unstable || is_doc_hidden {
+                            VariantVisibility::Hidden
+                        } else {
+                            VariantVisibility::Visible
+                        };
+                        variants[idx] = visibility;
+                    }
+
+                    ConstructorSet::Variants { variants, non_exhaustive: is_declared_nonexhaustive }
+                }
+            }
+            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => {
+                ConstructorSet::Single { empty: cx.is_uninhabited(ty) }
+            }
+            ty::Never => ConstructorSet::NoConstructors,
+            // This type is one for which we cannot list constructors, like `str` or `f64`.
+            // FIXME(Nadrieril): which of these are actually allowed?
+            ty::Float(_)
+            | ty::Str
+            | ty::Foreign(_)
+            | ty::RawPtr(_)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Closure(_, _)
+            | ty::Coroutine(_, _, _)
+            | ty::Alias(_, _)
+            | ty::Param(_)
+            | ty::Error(_) => ConstructorSet::Unlistable,
+            ty::CoroutineWitness(_, _) | ty::Bound(_, _) | ty::Placeholder(_) | ty::Infer(_) => {
+                bug!("Encountered unexpected type in `ConstructorSet::for_ty`: {ty:?}")
+            }
+        }
+    }
+
+    pub(crate) fn lower_pat_range_bdy(
+        &self,
+        bdy: PatRangeBoundary<'tcx>,
+        ty: Ty<'tcx>,
+    ) -> MaybeInfiniteInt {
+        match bdy {
+            PatRangeBoundary::NegInfinity => MaybeInfiniteInt::NegInfinity,
+            PatRangeBoundary::Finite(value) => {
+                let bits = value.eval_bits(self.tcx, self.param_env);
+                MaybeInfiniteInt::new_finite(self.tcx, ty, bits)
+            }
+            PatRangeBoundary::PosInfinity => MaybeInfiniteInt::PosInfinity,
+        }
+    }
+
+    /// Note: the input patterns must have been lowered through
+    /// `rustc_mir_build::thir::pattern::check_match::MatchVisitor::lower_pattern`.
+    pub fn lower_pat(&self, pat: &Pat<'tcx>) -> DeconstructedPat<'p, 'tcx> {
+        let singleton = |pat| std::slice::from_ref(self.pattern_arena.alloc(pat));
+        let cx = self;
+        let ctor;
+        let fields: &[_];
+        match &pat.kind {
+            PatKind::AscribeUserType { subpattern, .. }
+            | PatKind::InlineConstant { subpattern, .. } => return self.lower_pat(subpattern),
+            PatKind::Binding { subpattern: Some(subpat), .. } => return self.lower_pat(subpat),
+            PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
+                ctor = Wildcard;
+                fields = &[];
+            }
+            PatKind::Deref { subpattern } => {
+                ctor = Single;
+                fields = singleton(self.lower_pat(subpattern));
+            }
+            PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => {
+                match pat.ty.kind() {
+                    ty::Tuple(fs) => {
+                        ctor = Single;
+                        let mut wilds: SmallVec<[_; 2]> =
+                            fs.iter().map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
+                        for pat in subpatterns {
+                            wilds[pat.field.index()] = self.lower_pat(&pat.pattern);
+                        }
+                        fields = cx.pattern_arena.alloc_from_iter(wilds);
+                    }
+                    ty::Adt(adt, args) if adt.is_box() => {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
+                        // _)` or a box pattern. As a hack to avoid an ICE with the former, we
+                        // ignore other fields than the first one. This will trigger an error later
+                        // anyway.
+                        // See https://github.com/rust-lang/rust/issues/82772 ,
+                        // explanation: https://github.com/rust-lang/rust/pull/82789#issuecomment-796921977
+                        // The problem is that we can't know from the type whether we'll match
+                        // normally or through box-patterns. We'll have to figure out a proper
+                        // solution when we introduce generalized deref patterns. Also need to
+                        // prevent mixing of those two options.
+                        let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
+                        let pat = if let Some(pat) = pattern {
+                            self.lower_pat(&pat.pattern)
+                        } else {
+                            DeconstructedPat::wildcard(args.type_at(0), pat.span)
+                        };
+                        ctor = Single;
+                        fields = singleton(pat);
+                    }
+                    ty::Adt(adt, _) => {
+                        ctor = match pat.kind {
+                            PatKind::Leaf { .. } => Single,
+                            PatKind::Variant { variant_index, .. } => Variant(variant_index),
+                            _ => bug!(),
+                        };
+                        let variant =
+                            &adt.variant(MatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
+                        // For each field in the variant, we store the relevant index into `self.fields` if any.
+                        let mut field_id_to_id: Vec<Option<usize>> =
+                            (0..variant.fields.len()).map(|_| None).collect();
+                        let tys = cx
+                            .list_variant_nonhidden_fields(pat.ty, variant)
+                            .enumerate()
+                            .map(|(i, (field, ty))| {
+                                field_id_to_id[field.index()] = Some(i);
+                                ty
+                            });
+                        let mut wilds: SmallVec<[_; 2]> =
+                            tys.map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
+                        for pat in subpatterns {
+                            if let Some(i) = field_id_to_id[pat.field.index()] {
+                                wilds[i] = self.lower_pat(&pat.pattern);
+                            }
+                        }
+                        fields = cx.pattern_arena.alloc_from_iter(wilds);
+                    }
+                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
+                }
+            }
+            PatKind::Constant { value } => {
+                match pat.ty.kind() {
+                    ty::Bool => {
+                        ctor = match value.try_eval_bool(cx.tcx, cx.param_env) {
+                            Some(b) => Bool(b),
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Char | ty::Int(_) | ty::Uint(_) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => IntRange(IntRange::from_bits(cx.tcx, pat.ty, bits)),
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Float(ty::FloatTy::F32) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => {
+                                use rustc_apfloat::Float;
+                                let value = rustc_apfloat::ieee::Single::from_bits(bits);
+                                F32Range(value, value, RangeEnd::Included)
+                            }
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Float(ty::FloatTy::F64) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => {
+                                use rustc_apfloat::Float;
+                                let value = rustc_apfloat::ieee::Double::from_bits(bits);
+                                F64Range(value, value, RangeEnd::Included)
+                            }
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Ref(_, t, _) if t.is_str() => {
+                        // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
+                        // with other `Deref` patterns. This could have been done in `const_to_pat`,
+                        // but that causes issues with the rest of the matching code.
+                        // So here, the constructor for a `"foo"` pattern is `&` (represented by
+                        // `Single`), and has one field. That field has constructor `Str(value)` and no
+                        // fields.
+                        // Note: `t` is `str`, not `&str`.
+                        let subpattern = DeconstructedPat::new(Str(*value), &[], *t, pat.span);
+                        ctor = Single;
+                        fields = singleton(subpattern)
+                    }
+                    // All constants that can be structurally matched have already been expanded
+                    // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
+                    // opaque.
+                    _ => {
+                        ctor = Opaque(OpaqueId::new());
+                        fields = &[];
+                    }
+                }
+            }
+            PatKind::Range(patrange) => {
+                let PatRange { lo, hi, end, .. } = patrange.as_ref();
+                let ty = pat.ty;
+                ctor = match ty.kind() {
+                    ty::Char | ty::Int(_) | ty::Uint(_) => {
+                        let lo = cx.lower_pat_range_bdy(*lo, ty);
+                        let hi = cx.lower_pat_range_bdy(*hi, ty);
+                        IntRange(IntRange::from_range(lo, hi, *end))
+                    }
+                    ty::Float(fty) => {
+                        use rustc_apfloat::Float;
+                        let lo = lo.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        let hi = hi.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        match fty {
+                            ty::FloatTy::F32 => {
+                                use rustc_apfloat::ieee::Single;
+                                let lo = lo.map(Single::from_bits).unwrap_or(-Single::INFINITY);
+                                let hi = hi.map(Single::from_bits).unwrap_or(Single::INFINITY);
+                                F32Range(lo, hi, *end)
+                            }
+                            ty::FloatTy::F64 => {
+                                use rustc_apfloat::ieee::Double;
+                                let lo = lo.map(Double::from_bits).unwrap_or(-Double::INFINITY);
+                                let hi = hi.map(Double::from_bits).unwrap_or(Double::INFINITY);
+                                F64Range(lo, hi, *end)
+                            }
+                        }
+                    }
+                    _ => bug!("invalid type for range pattern: {}", ty),
+                };
+                fields = &[];
+            }
+            PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
+                let array_len = match pat.ty.kind() {
+                    ty::Array(_, length) => {
+                        Some(length.eval_target_usize(cx.tcx, cx.param_env) as usize)
+                    }
+                    ty::Slice(_) => None,
+                    _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
+                };
+                let kind = if slice.is_some() {
+                    SliceKind::VarLen(prefix.len(), suffix.len())
+                } else {
+                    SliceKind::FixedLen(prefix.len() + suffix.len())
+                };
+                ctor = Slice(Slice::new(array_len, kind));
+                fields = cx.pattern_arena.alloc_from_iter(
+                    prefix.iter().chain(suffix.iter()).map(|p| self.lower_pat(&*p)),
+                )
+            }
+            PatKind::Or { .. } => {
+                ctor = Or;
+                let pats = expand_or_pat(pat);
+                fields =
+                    cx.pattern_arena.alloc_from_iter(pats.into_iter().map(|p| self.lower_pat(p)))
+            }
+            PatKind::Never => {
+                // FIXME(never_patterns): handle `!` in exhaustiveness. This is a sane default
+                // in the meantime.
+                ctor = Wildcard;
+                fields = &[];
+            }
+            PatKind::Error(_) => {
+                ctor = Opaque(OpaqueId::new());
+                fields = &[];
+            }
+        }
+        DeconstructedPat::new(ctor, fields, pat.ty, pat.span)
+    }
+
+    /// Convert back to a `thir::PatRangeBoundary` for diagnostic purposes.
+    /// Note: it is possible to get `isize/usize::MAX+1` here, as explained in the doc for
+    /// [`IntRange::split`]. This cannot be represented as a `Const`, so we represent it with
+    /// `PosInfinity`.
+    pub(crate) fn hoist_pat_range_bdy(
+        &self,
+        miint: MaybeInfiniteInt,
+        ty: Ty<'tcx>,
+    ) -> PatRangeBoundary<'tcx> {
+        use MaybeInfiniteInt::*;
+        let tcx = self.tcx;
+        match miint {
+            NegInfinity => PatRangeBoundary::NegInfinity,
+            Finite(x) => {
+                let bias = MaybeInfiniteInt::signed_bias(tcx, ty);
+                let bits = x ^ bias;
+                let size = ty.primitive_size(tcx);
+                match Scalar::try_from_uint(bits, size) {
+                    Some(scalar) => {
+                        let value = mir::Const::from_scalar(tcx, scalar, ty);
+                        PatRangeBoundary::Finite(value)
+                    }
+                    // The value doesn't fit. Since `x >= 0` and 0 always encodes the minimum value
+                    // for a type, the problem isn't that the value is too small. So it must be too
+                    // large.
+                    None => PatRangeBoundary::PosInfinity,
+                }
+            }
+            JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
+        }
+    }
+
+    /// Whether the range denotes the fictitious values before `isize::MIN` or after
+    /// `usize::MAX`/`isize::MAX` (see doc of [`IntRange::split`] for why these exist).
+    pub fn is_range_beyond_boundaries(&self, range: &IntRange, ty: Ty<'tcx>) -> bool {
+        ty.is_ptr_sized_integral() && {
+            // The two invalid ranges are `NegInfinity..isize::MIN` (represented as
+            // `NegInfinity..0`), and `{u,i}size::MAX+1..PosInfinity`. `hoist_pat_range_bdy`
+            // converts `MAX+1` to `PosInfinity`, and we couldn't have `PosInfinity` in `range.lo`
+            // otherwise.
+            let lo = self.hoist_pat_range_bdy(range.lo, ty);
+            matches!(lo, PatRangeBoundary::PosInfinity)
+                || matches!(range.hi, MaybeInfiniteInt::Finite(0))
+        }
+    }
+
+    /// Convert back to a `thir::Pat` for diagnostic purposes.
+    pub(crate) fn hoist_pat_range(&self, range: &IntRange, ty: Ty<'tcx>) -> Pat<'tcx> {
+        use MaybeInfiniteInt::*;
+        let cx = self;
+        let kind = if matches!((range.lo, range.hi), (NegInfinity, PosInfinity)) {
+            PatKind::Wild
+        } else if range.is_singleton() {
+            let lo = cx.hoist_pat_range_bdy(range.lo, ty);
+            let value = lo.as_finite().unwrap();
+            PatKind::Constant { value }
+        } else {
+            // We convert to an inclusive range for diagnostics.
+            let mut end = RangeEnd::Included;
+            let mut lo = cx.hoist_pat_range_bdy(range.lo, ty);
+            if matches!(lo, PatRangeBoundary::PosInfinity) {
+                // The only reason to get `PosInfinity` here is the special case where
+                // `hoist_pat_range_bdy` found `{u,i}size::MAX+1`. So the range denotes the
+                // fictitious values after `{u,i}size::MAX` (see [`IntRange::split`] for why we do
+                // this). We show this to the user as `usize::MAX..` which is slightly incorrect but
+                // probably clear enough.
+                let c = ty.numeric_max_val(cx.tcx).unwrap();
+                let value = mir::Const::from_ty_const(c, cx.tcx);
+                lo = PatRangeBoundary::Finite(value);
+            }
+            let hi = if matches!(range.hi, Finite(0)) {
+                // The range encodes `..ty::MIN`, so we can't convert it to an inclusive range.
+                end = RangeEnd::Excluded;
+                range.hi
+            } else {
+                range.hi.minus_one()
+            };
+            let hi = cx.hoist_pat_range_bdy(hi, ty);
+            PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
+        };
+
+        Pat { ty, span: DUMMY_SP, kind }
+    }
+    /// Convert back to a `thir::Pat` for diagnostic purposes. This panics for patterns that don't
+    /// appear in diagnostics, like float ranges.
+    pub fn hoist_witness_pat(&self, pat: &WitnessPat<'tcx>) -> Pat<'tcx> {
+        let cx = self;
+        let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
+        let mut subpatterns = pat.iter_fields().map(|p| Box::new(cx.hoist_witness_pat(p)));
+        let kind = match pat.ctor() {
+            Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
+            IntRange(range) => return self.hoist_pat_range(range, pat.ty()),
+            Single | Variant(_) => match pat.ty().kind() {
+                ty::Tuple(..) => PatKind::Leaf {
+                    subpatterns: subpatterns
+                        .enumerate()
+                        .map(|(i, pattern)| FieldPat { field: FieldIdx::new(i), pattern })
+                        .collect(),
+                },
+                ty::Adt(adt_def, _) if adt_def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    PatKind::Deref { subpattern: subpatterns.next().unwrap() }
+                }
+                ty::Adt(adt_def, args) => {
+                    let variant_index =
+                        MatchCheckCtxt::variant_index_for_adt(&pat.ctor(), *adt_def);
+                    let variant = &adt_def.variant(variant_index);
+                    let subpatterns = cx
+                        .list_variant_nonhidden_fields(pat.ty(), variant)
+                        .zip(subpatterns)
+                        .map(|((field, _ty), pattern)| FieldPat { field, pattern })
+                        .collect();
+
+                    if adt_def.is_enum() {
+                        PatKind::Variant { adt_def: *adt_def, args, variant_index, subpatterns }
+                    } else {
+                        PatKind::Leaf { subpatterns }
+                    }
+                }
+                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+                // be careful to reconstruct the correct constant pattern here. However a string
+                // literal pattern will never be reported as a non-exhaustiveness witness, so we
+                // ignore this issue.
+                ty::Ref(..) => PatKind::Deref { subpattern: subpatterns.next().unwrap() },
+                _ => bug!("unexpected ctor for type {:?} {:?}", pat.ctor(), pat.ty()),
+            },
+            Slice(slice) => {
+                match slice.kind {
+                    SliceKind::FixedLen(_) => PatKind::Slice {
+                        prefix: subpatterns.collect(),
+                        slice: None,
+                        suffix: Box::new([]),
+                    },
+                    SliceKind::VarLen(prefix, _) => {
+                        let mut subpatterns = subpatterns.peekable();
+                        let mut prefix: Vec<_> = subpatterns.by_ref().take(prefix).collect();
+                        if slice.array_len.is_some() {
+                            // Improves diagnostics a bit: if the type is a known-size array, instead
+                            // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
+                            // This is incorrect if the size is not known, since `[_, ..]` captures
+                            // arrays of lengths `>= 1` whereas `[..]` captures any length.
+                            while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
+                                prefix.pop();
+                            }
+                            while subpatterns.peek().is_some()
+                                && is_wildcard(subpatterns.peek().unwrap())
+                            {
+                                subpatterns.next();
+                            }
+                        }
+                        let suffix: Box<[_]> = subpatterns.collect();
+                        let wild = Pat::wildcard_from_ty(pat.ty());
+                        PatKind::Slice {
+                            prefix: prefix.into_boxed_slice(),
+                            slice: Some(Box::new(wild)),
+                            suffix,
+                        }
+                    }
+                }
+            }
+            &Str(value) => PatKind::Constant { value },
+            Wildcard | NonExhaustive | Hidden => PatKind::Wild,
+            Missing { .. } => bug!(
+                "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
+                `Missing` should have been processed in `apply_constructors`"
+            ),
+            F32Range(..) | F64Range(..) | Opaque(..) | Or => {
+                bug!("can't convert to pattern: {:?}", pat)
+            }
+        };
+
+        Pat { ty: pat.ty(), span: DUMMY_SP, kind }
+    }
+
+    /// Best-effort `Debug` implementation.
+    pub(crate) fn debug_pat(
+        f: &mut fmt::Formatter<'_>,
+        pat: &DeconstructedPat<'p, 'tcx>,
+    ) -> fmt::Result {
+        let mut first = true;
+        let mut start_or_continue = |s| {
+            if first {
+                first = false;
+                ""
+            } else {
+                s
+            }
+        };
+        let mut start_or_comma = || start_or_continue(", ");
+
+        match pat.ctor() {
+            Single | Variant(_) => match pat.ty().kind() {
+                ty::Adt(def, _) if def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    let subpattern = pat.iter_fields().next().unwrap();
+                    write!(f, "box {subpattern:?}")
+                }
+                ty::Adt(..) | ty::Tuple(..) => {
+                    let variant = match pat.ty().kind() {
+                        ty::Adt(adt, _) => Some(
+                            adt.variant(MatchCheckCtxt::variant_index_for_adt(pat.ctor(), *adt)),
+                        ),
+                        ty::Tuple(_) => None,
+                        _ => unreachable!(),
+                    };
+
+                    if let Some(variant) = variant {
+                        write!(f, "{}", variant.name)?;
+                    }
+
+                    // Without `cx`, we can't know which field corresponds to which, so we can't
+                    // get the names of the fields. Instead we just display everything as a tuple
+                    // struct, which should be good enough.
+                    write!(f, "(")?;
+                    for p in pat.iter_fields() {
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "{p:?}")?;
+                    }
+                    write!(f, ")")
+                }
+                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+                // be careful to detect strings here. However a string literal pattern will never
+                // be reported as a non-exhaustiveness witness, so we can ignore this issue.
+                ty::Ref(_, _, mutbl) => {
+                    let subpattern = pat.iter_fields().next().unwrap();
+                    write!(f, "&{}{:?}", mutbl.prefix_str(), subpattern)
+                }
+                _ => write!(f, "_"),
+            },
+            Slice(slice) => {
+                let mut subpatterns = pat.iter_fields();
+                write!(f, "[")?;
+                match slice.kind {
+                    SliceKind::FixedLen(_) => {
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                    SliceKind::VarLen(prefix_len, _) => {
+                        for p in subpatterns.by_ref().take(prefix_len) {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "..")?;
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                }
+                write!(f, "]")
+            }
+            Bool(b) => write!(f, "{b}"),
+            // Best-effort, will render signed ranges incorrectly
+            IntRange(range) => write!(f, "{range:?}"),
+            F32Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
+            F64Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
+            Str(value) => write!(f, "{value}"),
+            Opaque(..) => write!(f, "<constant pattern>"),
+            Or => {
+                for pat in pat.iter_fields() {
+                    write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
+                }
+                Ok(())
+            }
+            Wildcard | Missing { .. } | NonExhaustive | Hidden => write!(f, "_ : {:?}", pat.ty()),
+        }
+    }
+}
+
+/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
+fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
+    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
+        if let PatKind::Or { pats } = &pat.kind {
+            for pat in pats.iter() {
+                expand(pat, vec);
+            }
+        } else {
+            vec.push(pat)
+        }
+    }
+
+    let mut pats = Vec::new();
+    expand(pat, &mut pats);
+    pats
+}
diff --git a/compiler/rustc_pattern_analysis/src/errors.rs b/compiler/rustc_pattern_analysis/src/errors.rs
index 0dddcb505e8..0efa8a0ec08 100644
--- a/compiler/rustc_pattern_analysis/src/errors.rs
+++ b/compiler/rustc_pattern_analysis/src/errors.rs
@@ -1,4 +1,4 @@
-use crate::{pat::WitnessPat, usefulness::MatchCheckCtxt};
+use crate::{cx::MatchCheckCtxt, pat::WitnessPat};
 
 use rustc_errors::{AddToDiagnostic, Diagnostic, SubdiagnosticMessage};
 use rustc_macros::{LintDiagnostic, Subdiagnostic};
@@ -24,18 +24,18 @@ impl<'tcx> Uncovered<'tcx> {
         cx: &MatchCheckCtxt<'p, 'tcx>,
         witnesses: Vec<WitnessPat<'tcx>>,
     ) -> Self {
-        let witness_1 = witnesses.get(0).unwrap().to_diagnostic_pat(cx);
+        let witness_1 = cx.hoist_witness_pat(witnesses.get(0).unwrap());
         Self {
             span,
             count: witnesses.len(),
             // Substitute dummy values if witnesses is smaller than 3. These will never be read.
             witness_2: witnesses
                 .get(1)
-                .map(|w| w.to_diagnostic_pat(cx))
+                .map(|w| cx.hoist_witness_pat(w))
                 .unwrap_or_else(|| witness_1.clone()),
             witness_3: witnesses
                 .get(2)
-                .map(|w| w.to_diagnostic_pat(cx))
+                .map(|w| cx.hoist_witness_pat(w))
                 .unwrap_or_else(|| witness_1.clone()),
             witness_1,
             remainder: witnesses.len().saturating_sub(3),
diff --git a/compiler/rustc_pattern_analysis/src/lib.rs b/compiler/rustc_pattern_analysis/src/lib.rs
index bf131a37809..5ccc0789c9e 100644
--- a/compiler/rustc_pattern_analysis/src/lib.rs
+++ b/compiler/rustc_pattern_analysis/src/lib.rs
@@ -1,6 +1,7 @@
 //! Analysis of patterns, notably match exhaustiveness checking.
 
 pub mod constructor;
+pub mod cx;
 pub mod errors;
 pub mod pat;
 pub mod usefulness;
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
index ded992cda93..bdbe14a08e3 100644
--- a/compiler/rustc_pattern_analysis/src/pat.rs
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -2,175 +2,19 @@
 //! fields. This file defines types that represent patterns in this way.
 use std::cell::Cell;
 use std::fmt;
-use std::iter::once;
 
 use smallvec::{smallvec, SmallVec};
 
 use rustc_data_structures::captures::Captures;
-use rustc_hir::RangeEnd;
-use rustc_index::Idx;
-use rustc_middle::mir;
-use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange};
-use rustc_middle::ty::{self, Ty, VariantDef};
+use rustc_middle::ty::{self, Ty};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::FieldIdx;
 
 use self::Constructor::*;
 use self::SliceKind::*;
 
-use crate::constructor::{Constructor, IntRange, MaybeInfiniteInt, OpaqueId, Slice, SliceKind};
-use crate::usefulness::{MatchCheckCtxt, PatCtxt};
-
-/// A value can be decomposed into a constructor applied to some fields. This struct represents
-/// those fields, generalized to allow patterns in each field. See also `Constructor`.
-///
-/// This is constructed for a constructor using [`Fields::wildcards()`]. The idea is that
-/// [`Fields::wildcards()`] constructs a list of fields where all entries are wildcards, and then
-/// given a pattern we fill some of the fields with its subpatterns.
-/// In the following example `Fields::wildcards` returns `[_, _, _, _]`. Then in
-/// `extract_pattern_arguments` we fill some of the entries, and the result is
-/// `[Some(0), _, _, _]`.
-/// ```compile_fail,E0004
-/// # fn foo() -> [Option<u8>; 4] { [None; 4] }
-/// let x: [Option<u8>; 4] = foo();
-/// match x {
-///     [Some(0), ..] => {}
-/// }
-/// ```
-///
-/// Note that the number of fields of a constructor may not match the fields declared in the
-/// original struct/variant. This happens if a private or `non_exhaustive` field is uninhabited,
-/// because the code mustn't observe that it is uninhabited. In that case that field is not
-/// included in `fields`. For that reason, when you have a `FieldIdx` you must use
-/// `index_with_declared_idx`.
-#[derive(Debug, Clone, Copy)]
-pub struct Fields<'p, 'tcx> {
-    fields: &'p [DeconstructedPat<'p, 'tcx>],
-}
-
-impl<'p, 'tcx> Fields<'p, 'tcx> {
-    fn empty() -> Self {
-        Fields { fields: &[] }
-    }
-
-    fn singleton(cx: &MatchCheckCtxt<'p, 'tcx>, field: DeconstructedPat<'p, 'tcx>) -> Self {
-        let field: &_ = cx.pattern_arena.alloc(field);
-        Fields { fields: std::slice::from_ref(field) }
-    }
-
-    pub fn from_iter(
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        fields: impl IntoIterator<Item = DeconstructedPat<'p, 'tcx>>,
-    ) -> Self {
-        let fields: &[_] = cx.pattern_arena.alloc_from_iter(fields);
-        Fields { fields }
-    }
-
-    fn wildcards_from_tys(
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        tys: impl IntoIterator<Item = Ty<'tcx>>,
-    ) -> Self {
-        Fields::from_iter(cx, tys.into_iter().map(|ty| DeconstructedPat::wildcard(ty, DUMMY_SP)))
-    }
-
-    // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
-    // uninhabited fields in order not to reveal the uninhabitedness of the whole variant.
-    // This lists the fields we keep along with their types.
-    pub(crate) fn list_variant_nonhidden_fields<'a>(
-        cx: &'a MatchCheckCtxt<'p, 'tcx>,
-        ty: Ty<'tcx>,
-        variant: &'a VariantDef,
-    ) -> impl Iterator<Item = (FieldIdx, Ty<'tcx>)> + Captures<'a> + Captures<'p> {
-        let ty::Adt(adt, args) = ty.kind() else { bug!() };
-        // Whether we must not match the fields of this variant exhaustively.
-        let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
-
-        variant.fields.iter().enumerate().filter_map(move |(i, field)| {
-            let ty = field.ty(cx.tcx, args);
-            // `field.ty()` doesn't normalize after substituting.
-            let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
-            let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
-            let is_uninhabited = cx.tcx.features().exhaustive_patterns && cx.is_uninhabited(ty);
-
-            if is_uninhabited && (!is_visible || is_non_exhaustive) {
-                None
-            } else {
-                Some((FieldIdx::new(i), ty))
-            }
-        })
-    }
-
-    /// Creates a new list of wildcard fields for a given constructor. The result must have a
-    /// length of `constructor.arity()`.
-    #[instrument(level = "trace")]
-    pub(super) fn wildcards(pcx: &PatCtxt<'_, 'p, 'tcx>, constructor: &Constructor<'tcx>) -> Self {
-        let ret = match constructor {
-            Single | Variant(_) => match pcx.ty.kind() {
-                ty::Tuple(fs) => Fields::wildcards_from_tys(pcx.cx, fs.iter()),
-                ty::Ref(_, rty, _) => Fields::wildcards_from_tys(pcx.cx, once(*rty)),
-                ty::Adt(adt, args) => {
-                    if adt.is_box() {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        Fields::wildcards_from_tys(pcx.cx, once(args.type_at(0)))
-                    } else {
-                        let variant = &adt.variant(constructor.variant_index_for_adt(*adt));
-                        let tys = Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant)
-                            .map(|(_, ty)| ty);
-                        Fields::wildcards_from_tys(pcx.cx, tys)
-                    }
-                }
-                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx),
-            },
-            Slice(slice) => match *pcx.ty.kind() {
-                ty::Slice(ty) | ty::Array(ty, _) => {
-                    let arity = slice.arity();
-                    Fields::wildcards_from_tys(pcx.cx, (0..arity).map(|_| ty))
-                }
-                _ => bug!("bad slice pattern {:?} {:?}", constructor, pcx),
-            },
-            Bool(..)
-            | IntRange(..)
-            | F32Range(..)
-            | F64Range(..)
-            | Str(..)
-            | Opaque(..)
-            | NonExhaustive
-            | Hidden
-            | Missing { .. }
-            | Wildcard => Fields::empty(),
-            Or => {
-                bug!("called `Fields::wildcards` on an `Or` ctor")
-            }
-        };
-        debug!(?ret);
-        ret
-    }
-
-    /// Returns the list of patterns.
-    pub(super) fn iter_patterns<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.fields.iter()
-    }
-}
-
-/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
-fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
-    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
-        if let PatKind::Or { pats } = &pat.kind {
-            for pat in pats.iter() {
-                expand(pat, vec);
-            }
-        } else {
-            vec.push(pat)
-        }
-    }
-
-    let mut pats = Vec::new();
-    expand(pat, &mut pats);
-    pats
-}
+use crate::constructor::{Constructor, SliceKind};
+use crate::cx::MatchCheckCtxt;
+use crate::usefulness::PatCtxt;
 
 /// Values and patterns can be represented as a constructor applied to some fields. This represents
 /// a pattern in this form.
@@ -178,9 +22,14 @@ fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
 /// during analysis. For this reason they cannot be cloned.
 /// A `DeconstructedPat` will almost always come from user input; the only exception are some
 /// `Wildcard`s introduced during specialization.
+///
+/// Note that the number of fields may not match the fields declared in the original struct/variant.
+/// This happens if a private or `non_exhaustive` field is uninhabited, because the code mustn't
+/// observe that it is uninhabited. In that case that field is not included in `fields`. Care must
+/// be taken when converting to/from `thir::Pat`.
 pub struct DeconstructedPat<'p, 'tcx> {
     ctor: Constructor<'tcx>,
-    fields: Fields<'p, 'tcx>,
+    fields: &'p [DeconstructedPat<'p, 'tcx>],
     ty: Ty<'tcx>,
     span: Span,
     /// Whether removing this arm would change the behavior of the match expression.
@@ -189,227 +38,18 @@ pub struct DeconstructedPat<'p, 'tcx> {
 
 impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
     pub(super) fn wildcard(ty: Ty<'tcx>, span: Span) -> Self {
-        Self::new(Wildcard, Fields::empty(), ty, span)
+        Self::new(Wildcard, &[], ty, span)
     }
 
     pub(super) fn new(
         ctor: Constructor<'tcx>,
-        fields: Fields<'p, 'tcx>,
+        fields: &'p [DeconstructedPat<'p, 'tcx>],
         ty: Ty<'tcx>,
         span: Span,
     ) -> Self {
         DeconstructedPat { ctor, fields, ty, span, useful: Cell::new(false) }
     }
 
-    /// Note: the input patterns must have been lowered through
-    /// `rustc_mir_build::thir::pattern::check_match::MatchVisitor::lower_pattern`.
-    pub fn from_pat(cx: &MatchCheckCtxt<'p, 'tcx>, pat: &Pat<'tcx>) -> Self {
-        let mkpat = |pat| DeconstructedPat::from_pat(cx, pat);
-        let ctor;
-        let fields;
-        match &pat.kind {
-            PatKind::AscribeUserType { subpattern, .. }
-            | PatKind::InlineConstant { subpattern, .. } => return mkpat(subpattern),
-            PatKind::Binding { subpattern: Some(subpat), .. } => return mkpat(subpat),
-            PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
-                ctor = Wildcard;
-                fields = Fields::empty();
-            }
-            PatKind::Deref { subpattern } => {
-                ctor = Single;
-                fields = Fields::singleton(cx, mkpat(subpattern));
-            }
-            PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => {
-                match pat.ty.kind() {
-                    ty::Tuple(fs) => {
-                        ctor = Single;
-                        let mut wilds: SmallVec<[_; 2]> =
-                            fs.iter().map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
-                        for pat in subpatterns {
-                            wilds[pat.field.index()] = mkpat(&pat.pattern);
-                        }
-                        fields = Fields::from_iter(cx, wilds);
-                    }
-                    ty::Adt(adt, args) if adt.is_box() => {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
-                        // _)` or a box pattern. As a hack to avoid an ICE with the former, we
-                        // ignore other fields than the first one. This will trigger an error later
-                        // anyway.
-                        // See https://github.com/rust-lang/rust/issues/82772 ,
-                        // explanation: https://github.com/rust-lang/rust/pull/82789#issuecomment-796921977
-                        // The problem is that we can't know from the type whether we'll match
-                        // normally or through box-patterns. We'll have to figure out a proper
-                        // solution when we introduce generalized deref patterns. Also need to
-                        // prevent mixing of those two options.
-                        let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
-                        let pat = if let Some(pat) = pattern {
-                            mkpat(&pat.pattern)
-                        } else {
-                            DeconstructedPat::wildcard(args.type_at(0), pat.span)
-                        };
-                        ctor = Single;
-                        fields = Fields::singleton(cx, pat);
-                    }
-                    ty::Adt(adt, _) => {
-                        ctor = match pat.kind {
-                            PatKind::Leaf { .. } => Single,
-                            PatKind::Variant { variant_index, .. } => Variant(variant_index),
-                            _ => bug!(),
-                        };
-                        let variant = &adt.variant(ctor.variant_index_for_adt(*adt));
-                        // For each field in the variant, we store the relevant index into `self.fields` if any.
-                        let mut field_id_to_id: Vec<Option<usize>> =
-                            (0..variant.fields.len()).map(|_| None).collect();
-                        let tys = Fields::list_variant_nonhidden_fields(cx, pat.ty, variant)
-                            .enumerate()
-                            .map(|(i, (field, ty))| {
-                                field_id_to_id[field.index()] = Some(i);
-                                ty
-                            });
-                        let mut wilds: SmallVec<[_; 2]> =
-                            tys.map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
-                        for pat in subpatterns {
-                            if let Some(i) = field_id_to_id[pat.field.index()] {
-                                wilds[i] = mkpat(&pat.pattern);
-                            }
-                        }
-                        fields = Fields::from_iter(cx, wilds);
-                    }
-                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
-                }
-            }
-            PatKind::Constant { value } => {
-                match pat.ty.kind() {
-                    ty::Bool => {
-                        ctor = match value.try_eval_bool(cx.tcx, cx.param_env) {
-                            Some(b) => Bool(b),
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Char | ty::Int(_) | ty::Uint(_) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => IntRange(IntRange::from_bits(cx.tcx, pat.ty, bits)),
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Float(ty::FloatTy::F32) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => {
-                                use rustc_apfloat::Float;
-                                let value = rustc_apfloat::ieee::Single::from_bits(bits);
-                                F32Range(value, value, RangeEnd::Included)
-                            }
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Float(ty::FloatTy::F64) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => {
-                                use rustc_apfloat::Float;
-                                let value = rustc_apfloat::ieee::Double::from_bits(bits);
-                                F64Range(value, value, RangeEnd::Included)
-                            }
-                            None => Opaque(OpaqueId::new()),
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Ref(_, t, _) if t.is_str() => {
-                        // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
-                        // with other `Deref` patterns. This could have been done in `const_to_pat`,
-                        // but that causes issues with the rest of the matching code.
-                        // So here, the constructor for a `"foo"` pattern is `&` (represented by
-                        // `Single`), and has one field. That field has constructor `Str(value)` and no
-                        // fields.
-                        // Note: `t` is `str`, not `&str`.
-                        let subpattern =
-                            DeconstructedPat::new(Str(*value), Fields::empty(), *t, pat.span);
-                        ctor = Single;
-                        fields = Fields::singleton(cx, subpattern)
-                    }
-                    // All constants that can be structurally matched have already been expanded
-                    // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
-                    // opaque.
-                    _ => {
-                        ctor = Opaque(OpaqueId::new());
-                        fields = Fields::empty();
-                    }
-                }
-            }
-            PatKind::Range(patrange) => {
-                let PatRange { lo, hi, end, .. } = patrange.as_ref();
-                let ty = pat.ty;
-                ctor = match ty.kind() {
-                    ty::Char | ty::Int(_) | ty::Uint(_) => {
-                        let lo =
-                            MaybeInfiniteInt::from_pat_range_bdy(*lo, ty, cx.tcx, cx.param_env);
-                        let hi =
-                            MaybeInfiniteInt::from_pat_range_bdy(*hi, ty, cx.tcx, cx.param_env);
-                        IntRange(IntRange::from_range(lo, hi, *end))
-                    }
-                    ty::Float(fty) => {
-                        use rustc_apfloat::Float;
-                        let lo = lo.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
-                        let hi = hi.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
-                        match fty {
-                            ty::FloatTy::F32 => {
-                                use rustc_apfloat::ieee::Single;
-                                let lo = lo.map(Single::from_bits).unwrap_or(-Single::INFINITY);
-                                let hi = hi.map(Single::from_bits).unwrap_or(Single::INFINITY);
-                                F32Range(lo, hi, *end)
-                            }
-                            ty::FloatTy::F64 => {
-                                use rustc_apfloat::ieee::Double;
-                                let lo = lo.map(Double::from_bits).unwrap_or(-Double::INFINITY);
-                                let hi = hi.map(Double::from_bits).unwrap_or(Double::INFINITY);
-                                F64Range(lo, hi, *end)
-                            }
-                        }
-                    }
-                    _ => bug!("invalid type for range pattern: {}", ty),
-                };
-                fields = Fields::empty();
-            }
-            PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
-                let array_len = match pat.ty.kind() {
-                    ty::Array(_, length) => {
-                        Some(length.eval_target_usize(cx.tcx, cx.param_env) as usize)
-                    }
-                    ty::Slice(_) => None,
-                    _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
-                };
-                let kind = if slice.is_some() {
-                    VarLen(prefix.len(), suffix.len())
-                } else {
-                    FixedLen(prefix.len() + suffix.len())
-                };
-                ctor = Slice(Slice::new(array_len, kind));
-                fields =
-                    Fields::from_iter(cx, prefix.iter().chain(suffix.iter()).map(|p| mkpat(&*p)));
-            }
-            PatKind::Or { .. } => {
-                ctor = Or;
-                let pats = expand_or_pat(pat);
-                fields = Fields::from_iter(cx, pats.into_iter().map(mkpat));
-            }
-            PatKind::Never => {
-                // FIXME(never_patterns): handle `!` in exhaustiveness. This is a sane default
-                // in the meantime.
-                ctor = Wildcard;
-                fields = Fields::empty();
-            }
-            PatKind::Error(_) => {
-                ctor = Opaque(OpaqueId::new());
-                fields = Fields::empty();
-            }
-        }
-        DeconstructedPat::new(ctor, fields, pat.ty, pat.span)
-    }
-
     pub(super) fn is_or_pat(&self) -> bool {
         matches!(self.ctor, Or)
     }
@@ -435,7 +75,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
     pub fn iter_fields<'a>(
         &'a self,
     ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.fields.iter_patterns()
+        self.fields.iter()
     }
 
     /// Specialize this pattern with a constructor.
@@ -448,7 +88,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
         match (&self.ctor, other_ctor) {
             (Wildcard, _) => {
                 // We return a wildcard for each field of `other_ctor`.
-                Fields::wildcards(pcx, other_ctor).iter_patterns().collect()
+                pcx.cx.ctor_wildcard_fields(other_ctor, pcx.ty).iter().collect()
             }
             (Slice(self_slice), Slice(other_slice))
                 if self_slice.arity() != other_slice.arity() =>
@@ -464,8 +104,8 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                         let (ty::Slice(inner_ty) | ty::Array(inner_ty, _)) = *self.ty.kind() else {
                             bug!("bad slice pattern {:?} {:?}", self.ctor, self.ty);
                         };
-                        let prefix = &self.fields.fields[..prefix];
-                        let suffix = &self.fields.fields[self_slice.arity() - suffix..];
+                        let prefix = &self.fields[..prefix];
+                        let suffix = &self.fields[self_slice.arity() - suffix..];
                         let wildcard: &_ = pcx
                             .cx
                             .pattern_arena
@@ -476,7 +116,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                     }
                 }
             }
-            _ => self.fields.iter_patterns().collect(),
+            _ => self.fields.iter().collect(),
         }
     }
 
@@ -521,94 +161,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
 /// `Display` impl.
 impl<'p, 'tcx> fmt::Debug for DeconstructedPat<'p, 'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // Printing lists is a chore.
-        let mut first = true;
-        let mut start_or_continue = |s| {
-            if first {
-                first = false;
-                ""
-            } else {
-                s
-            }
-        };
-        let mut start_or_comma = || start_or_continue(", ");
-
-        match &self.ctor {
-            Single | Variant(_) => match self.ty.kind() {
-                ty::Adt(def, _) if def.is_box() => {
-                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
-                    // of `std`). So this branch is only reachable when the feature is enabled and
-                    // the pattern is a box pattern.
-                    let subpattern = self.iter_fields().next().unwrap();
-                    write!(f, "box {subpattern:?}")
-                }
-                ty::Adt(..) | ty::Tuple(..) => {
-                    let variant = match self.ty.kind() {
-                        ty::Adt(adt, _) => Some(adt.variant(self.ctor.variant_index_for_adt(*adt))),
-                        ty::Tuple(_) => None,
-                        _ => unreachable!(),
-                    };
-
-                    if let Some(variant) = variant {
-                        write!(f, "{}", variant.name)?;
-                    }
-
-                    // Without `cx`, we can't know which field corresponds to which, so we can't
-                    // get the names of the fields. Instead we just display everything as a tuple
-                    // struct, which should be good enough.
-                    write!(f, "(")?;
-                    for p in self.iter_fields() {
-                        write!(f, "{}", start_or_comma())?;
-                        write!(f, "{p:?}")?;
-                    }
-                    write!(f, ")")
-                }
-                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
-                // be careful to detect strings here. However a string literal pattern will never
-                // be reported as a non-exhaustiveness witness, so we can ignore this issue.
-                ty::Ref(_, _, mutbl) => {
-                    let subpattern = self.iter_fields().next().unwrap();
-                    write!(f, "&{}{:?}", mutbl.prefix_str(), subpattern)
-                }
-                _ => write!(f, "_"),
-            },
-            Slice(slice) => {
-                let mut subpatterns = self.fields.iter_patterns();
-                write!(f, "[")?;
-                match slice.kind {
-                    FixedLen(_) => {
-                        for p in subpatterns {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                    }
-                    VarLen(prefix_len, _) => {
-                        for p in subpatterns.by_ref().take(prefix_len) {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                        write!(f, "{}", start_or_comma())?;
-                        write!(f, "..")?;
-                        for p in subpatterns {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                    }
-                }
-                write!(f, "]")
-            }
-            Bool(b) => write!(f, "{b}"),
-            // Best-effort, will render signed ranges incorrectly
-            IntRange(range) => write!(f, "{range:?}"),
-            F32Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
-            F64Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
-            Str(value) => write!(f, "{value}"),
-            Opaque(..) => write!(f, "<constant pattern>"),
-            Or => {
-                for pat in self.iter_fields() {
-                    write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
-                }
-                Ok(())
-            }
-            Wildcard | Missing { .. } | NonExhaustive | Hidden => write!(f, "_ : {:?}", self.ty),
-        }
+        MatchCheckCtxt::debug_pat(f, self)
     }
 }
 
@@ -633,11 +186,9 @@ impl<'tcx> WitnessPat<'tcx> {
     /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
     /// `Some(_)`.
     pub(super) fn wild_from_ctor(pcx: &PatCtxt<'_, '_, 'tcx>, ctor: Constructor<'tcx>) -> Self {
-        // Reuse `Fields::wildcards` to get the types.
-        let fields = Fields::wildcards(pcx, &ctor)
-            .iter_patterns()
-            .map(|deco_pat| Self::wildcard(deco_pat.ty()))
-            .collect();
+        let field_tys =
+            pcx.cx.ctor_wildcard_fields(&ctor, pcx.ty).iter().map(|deco_pat| deco_pat.ty());
+        let fields = field_tys.map(|ty| Self::wildcard(ty)).collect();
         Self::new(ctor, fields, pcx.ty)
     }
 
@@ -648,96 +199,6 @@ impl<'tcx> WitnessPat<'tcx> {
         self.ty
     }
 
-    /// Convert back to a `thir::Pat` for diagnostic purposes. This panics for patterns that don't
-    /// appear in diagnostics, like float ranges.
-    pub fn to_diagnostic_pat(&self, cx: &MatchCheckCtxt<'_, 'tcx>) -> Pat<'tcx> {
-        let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
-        let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_diagnostic_pat(cx)));
-        let kind = match &self.ctor {
-            Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
-            IntRange(range) => return range.to_diagnostic_pat(self.ty, cx.tcx),
-            Single | Variant(_) => match self.ty.kind() {
-                ty::Tuple(..) => PatKind::Leaf {
-                    subpatterns: subpatterns
-                        .enumerate()
-                        .map(|(i, pattern)| FieldPat { field: FieldIdx::new(i), pattern })
-                        .collect(),
-                },
-                ty::Adt(adt_def, _) if adt_def.is_box() => {
-                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
-                    // of `std`). So this branch is only reachable when the feature is enabled and
-                    // the pattern is a box pattern.
-                    PatKind::Deref { subpattern: subpatterns.next().unwrap() }
-                }
-                ty::Adt(adt_def, args) => {
-                    let variant_index = self.ctor.variant_index_for_adt(*adt_def);
-                    let variant = &adt_def.variant(variant_index);
-                    let subpatterns = Fields::list_variant_nonhidden_fields(cx, self.ty, variant)
-                        .zip(subpatterns)
-                        .map(|((field, _ty), pattern)| FieldPat { field, pattern })
-                        .collect();
-
-                    if adt_def.is_enum() {
-                        PatKind::Variant { adt_def: *adt_def, args, variant_index, subpatterns }
-                    } else {
-                        PatKind::Leaf { subpatterns }
-                    }
-                }
-                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
-                // be careful to reconstruct the correct constant pattern here. However a string
-                // literal pattern will never be reported as a non-exhaustiveness witness, so we
-                // ignore this issue.
-                ty::Ref(..) => PatKind::Deref { subpattern: subpatterns.next().unwrap() },
-                _ => bug!("unexpected ctor for type {:?} {:?}", self.ctor, self.ty),
-            },
-            Slice(slice) => {
-                match slice.kind {
-                    FixedLen(_) => PatKind::Slice {
-                        prefix: subpatterns.collect(),
-                        slice: None,
-                        suffix: Box::new([]),
-                    },
-                    VarLen(prefix, _) => {
-                        let mut subpatterns = subpatterns.peekable();
-                        let mut prefix: Vec<_> = subpatterns.by_ref().take(prefix).collect();
-                        if slice.array_len.is_some() {
-                            // Improves diagnostics a bit: if the type is a known-size array, instead
-                            // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
-                            // This is incorrect if the size is not known, since `[_, ..]` captures
-                            // arrays of lengths `>= 1` whereas `[..]` captures any length.
-                            while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
-                                prefix.pop();
-                            }
-                            while subpatterns.peek().is_some()
-                                && is_wildcard(subpatterns.peek().unwrap())
-                            {
-                                subpatterns.next();
-                            }
-                        }
-                        let suffix: Box<[_]> = subpatterns.collect();
-                        let wild = Pat::wildcard_from_ty(self.ty);
-                        PatKind::Slice {
-                            prefix: prefix.into_boxed_slice(),
-                            slice: Some(Box::new(wild)),
-                            suffix,
-                        }
-                    }
-                }
-            }
-            &Str(value) => PatKind::Constant { value },
-            Wildcard | NonExhaustive | Hidden => PatKind::Wild,
-            Missing { .. } => bug!(
-                "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
-                `Missing` should have been processed in `apply_constructors`"
-            ),
-            F32Range(..) | F64Range(..) | Opaque(..) | Or => {
-                bug!("can't convert to pattern: {:?}", self)
-            }
-        };
-
-        Pat { ty: self.ty, span: DUMMY_SP, kind }
-    }
-
     pub fn iter_fields<'a>(&'a self) -> impl Iterator<Item = &'a WitnessPat<'tcx>> {
         self.fields.iter()
     }
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
index 5554f3fc36c..9cc48393fdf 100644
--- a/compiler/rustc_pattern_analysis/src/usefulness.rs
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -551,66 +551,27 @@
 //! I (Nadrieril) prefer to put new tests in `ui/pattern/usefulness` unless there's a specific
 //! reason not to, for example if they crucially depend on a particular feature like `or_patterns`.
 
-use self::ValidityConstraint::*;
+use smallvec::{smallvec, SmallVec};
+use std::fmt;
+
+use rustc_data_structures::{captures::Captures, stack::ensure_sufficient_stack};
+use rustc_hir::HirId;
+use rustc_middle::ty::{self, Ty};
+use rustc_session::lint;
+use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
+use rustc_span::{Span, DUMMY_SP};
+
 use crate::constructor::{
     Constructor, ConstructorSet, IntRange, MaybeInfiniteInt, SplitConstructorSet,
 };
+use crate::cx::MatchCheckCtxt;
 use crate::errors::{
     NonExhaustiveOmittedPattern, NonExhaustiveOmittedPatternLintOnArm, Overlap,
     OverlappingRangeEndpoints, Uncovered,
 };
 use crate::pat::{DeconstructedPat, WitnessPat};
 
-use rustc_arena::TypedArena;
-use rustc_data_structures::{captures::Captures, stack::ensure_sufficient_stack};
-use rustc_hir::def_id::DefId;
-use rustc_hir::HirId;
-use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_session::lint;
-use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
-use rustc_span::{Span, DUMMY_SP};
-
-use smallvec::{smallvec, SmallVec};
-use std::fmt;
-
-pub struct MatchCheckCtxt<'p, 'tcx> {
-    pub tcx: TyCtxt<'tcx>,
-    /// The module in which the match occurs. This is necessary for
-    /// checking inhabited-ness of types because whether a type is (visibly)
-    /// inhabited can depend on whether it was defined in the current module or
-    /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
-    /// outside its module and should not be matchable with an empty match statement.
-    pub module: DefId,
-    pub param_env: ty::ParamEnv<'tcx>,
-    pub pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
-    /// Lint level at the match.
-    pub match_lint_level: HirId,
-    /// The span of the whole match, if applicable.
-    pub whole_match_span: Option<Span>,
-    /// Span of the scrutinee.
-    pub scrut_span: Span,
-    /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
-    pub refutable: bool,
-    /// Whether the data at the scrutinee is known to be valid. This is false if the scrutinee comes
-    /// from a union field, a pointer deref, or a reference deref (pending opsem decisions).
-    pub known_valid_scrutinee: bool,
-}
-
-impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
-    pub(super) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
-        !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
-    }
-
-    /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
-    pub fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
-        match ty.kind() {
-            ty::Adt(def, ..) => {
-                def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did().is_local()
-            }
-            _ => false,
-        }
-    }
-}
+use self::ValidityConstraint::*;
 
 #[derive(Copy, Clone)]
 pub(super) struct PatCtxt<'a, 'p, 'tcx> {
@@ -1244,7 +1205,9 @@ fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
 
     // Analyze the constructors present in this column.
     let ctors = matrix.heads().map(|p| p.ctor());
-    let split_set = ConstructorSet::for_ty(cx, ty).split(pcx, ctors);
+    let ctors_for_ty = &cx.ctors_for_ty(ty);
+    let is_integers = matches!(ctors_for_ty, ConstructorSet::Integers { .. }); // For diagnostics.
+    let split_set = ctors_for_ty.split(pcx, ctors);
     let all_missing = split_set.present.is_empty();
 
     // Build the set of constructors we will specialize with. It must cover the whole type.
@@ -1259,7 +1222,7 @@ fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
     }
 
     // Decide what constructors to report.
-    let always_report_all = is_top_level && !IntRange::is_integral(pcx.ty);
+    let always_report_all = is_top_level && !is_integers;
     // Whether we should report "Enum::A and Enum::C are missing" or "_ is missing".
     let report_individual_missing_ctors = always_report_all || !all_missing;
     // Which constructors are considered missing. We ensure that `!missing_ctors.is_empty() =>
@@ -1362,7 +1325,7 @@ impl<'p, 'tcx> PatternColumn<'p, 'tcx> {
     /// Do constructor splitting on the constructors of the column.
     fn analyze_ctors(&self, pcx: &PatCtxt<'_, 'p, 'tcx>) -> SplitConstructorSet<'tcx> {
         let column_ctors = self.patterns.iter().map(|p| p.ctor());
-        ConstructorSet::for_ty(pcx.cx, pcx.ty).split(pcx, column_ctors)
+        pcx.cx.ctors_for_ty(pcx.ty).split(pcx, column_ctors)
     }
 
     fn iter<'a>(&'a self) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
@@ -1470,9 +1433,9 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
 
     let set = column.analyze_ctors(pcx);
 
-    if IntRange::is_integral(ty) {
+    if matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_)) {
         let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
-            let overlap_as_pat = overlap.to_diagnostic_pat(ty, cx.tcx);
+            let overlap_as_pat = cx.hoist_pat_range(overlap, ty);
             let overlaps: Vec<_> = overlapped_spans
                 .iter()
                 .copied()
-- 
cgit 1.4.1-3-g733a5


From 24adca0a26c3470cc0feabb19a77eb4e2e7b8278 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 11 Dec 2023 10:40:31 +0100
Subject: Move lints to their own module

---
 .../src/thir/pattern/check_match.rs                |   9 +-
 compiler/rustc_pattern_analysis/src/lib.rs         |  42 +++
 compiler/rustc_pattern_analysis/src/lints.rs       | 291 +++++++++++++++++++
 compiler/rustc_pattern_analysis/src/usefulness.rs  | 307 +--------------------
 4 files changed, 347 insertions(+), 302 deletions(-)
 create mode 100644 compiler/rustc_pattern_analysis/src/lints.rs

diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 02becbead66..e8a972005b0 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -2,9 +2,8 @@ use rustc_pattern_analysis::constructor::Constructor;
 use rustc_pattern_analysis::cx::MatchCheckCtxt;
 use rustc_pattern_analysis::errors::Uncovered;
 use rustc_pattern_analysis::pat::{DeconstructedPat, WitnessPat};
-use rustc_pattern_analysis::usefulness::{
-    compute_match_usefulness, MatchArm, Usefulness, UsefulnessReport,
-};
+use rustc_pattern_analysis::usefulness::{Usefulness, UsefulnessReport};
+use rustc_pattern_analysis::{analyze_match, MatchArm};
 
 use crate::errors::*;
 
@@ -436,7 +435,7 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         }
 
         let scrut_ty = scrut.ty;
-        let report = compute_match_usefulness(&cx, &tarms, scrut_ty);
+        let report = analyze_match(&cx, &tarms, scrut_ty);
 
         match source {
             // Don't report arm reachability of desugared `match $iter.into_iter() { iter => .. }`
@@ -550,7 +549,7 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         let cx = self.new_cx(refutability, None, scrut, pat.span);
         let pat = self.lower_pattern(&cx, pat)?;
         let arms = [MatchArm { pat, hir_id: self.lint_level, has_guard: false }];
-        let report = compute_match_usefulness(&cx, &arms, pat.ty());
+        let report = analyze_match(&cx, &arms, pat.ty());
         Ok((cx, report))
     }
 
diff --git a/compiler/rustc_pattern_analysis/src/lib.rs b/compiler/rustc_pattern_analysis/src/lib.rs
index 5ccc0789c9e..07730aa49d3 100644
--- a/compiler/rustc_pattern_analysis/src/lib.rs
+++ b/compiler/rustc_pattern_analysis/src/lib.rs
@@ -3,6 +3,7 @@
 pub mod constructor;
 pub mod cx;
 pub mod errors;
+pub(crate) mod lints;
 pub mod pat;
 pub mod usefulness;
 
@@ -12,3 +13,44 @@ extern crate tracing;
 extern crate rustc_middle;
 
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
+
+use lints::PatternColumn;
+use rustc_hir::HirId;
+use rustc_middle::ty::Ty;
+use usefulness::{compute_match_usefulness, UsefulnessReport};
+
+use crate::cx::MatchCheckCtxt;
+use crate::lints::{lint_nonexhaustive_missing_variants, lint_overlapping_range_endpoints};
+use crate::pat::DeconstructedPat;
+
+/// The arm of a match expression.
+#[derive(Clone, Copy, Debug)]
+pub struct MatchArm<'p, 'tcx> {
+    /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`.
+    pub pat: &'p DeconstructedPat<'p, 'tcx>,
+    pub hir_id: HirId,
+    pub has_guard: bool,
+}
+
+/// The entrypoint for this crate. Computes whether a match is exhaustive and which of its arms are
+/// useful, and runs some lints.
+pub fn analyze_match<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    arms: &[MatchArm<'p, 'tcx>],
+    scrut_ty: Ty<'tcx>,
+) -> UsefulnessReport<'p, 'tcx> {
+    let pat_column = PatternColumn::new(arms);
+
+    let report = compute_match_usefulness(cx, arms, scrut_ty);
+
+    // Lint on ranges that overlap on their endpoints, which is likely a mistake.
+    lint_overlapping_range_endpoints(cx, &pat_column);
+
+    // Run the non_exhaustive_omitted_patterns lint. Only run on refutable patterns to avoid hitting
+    // `if let`s. Only run if the match is exhaustive otherwise the error is redundant.
+    if cx.refutable && report.non_exhaustiveness_witnesses.is_empty() {
+        lint_nonexhaustive_missing_variants(cx, arms, &pat_column, scrut_ty)
+    }
+
+    report
+}
diff --git a/compiler/rustc_pattern_analysis/src/lints.rs b/compiler/rustc_pattern_analysis/src/lints.rs
new file mode 100644
index 00000000000..130945870e4
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/lints.rs
@@ -0,0 +1,291 @@
+use smallvec::SmallVec;
+
+use rustc_data_structures::captures::Captures;
+use rustc_middle::ty::{self, Ty};
+use rustc_session::lint;
+use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
+use rustc_span::Span;
+
+use crate::constructor::{Constructor, IntRange, MaybeInfiniteInt, SplitConstructorSet};
+use crate::cx::MatchCheckCtxt;
+use crate::errors::{
+    NonExhaustiveOmittedPattern, NonExhaustiveOmittedPatternLintOnArm, Overlap,
+    OverlappingRangeEndpoints, Uncovered,
+};
+use crate::pat::{DeconstructedPat, WitnessPat};
+use crate::usefulness::PatCtxt;
+use crate::MatchArm;
+
+/// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
+/// inspect the same subvalue/place".
+/// This is used to traverse patterns column-by-column for lints. Despite similarities with
+/// [`compute_exhaustiveness_and_usefulness`], this does a different traversal. Notably this is
+/// linear in the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case
+/// exponential (exhaustiveness is NP-complete). The core difference is that we treat sub-columns
+/// separately.
+///
+/// This must not contain an or-pattern. `specialize` takes care to expand them.
+///
+/// This is not used in the main algorithm; only in lints.
+#[derive(Debug)]
+pub(crate) struct PatternColumn<'p, 'tcx> {
+    patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>,
+}
+
+impl<'p, 'tcx> PatternColumn<'p, 'tcx> {
+    pub(crate) fn new(arms: &[MatchArm<'p, 'tcx>]) -> Self {
+        let mut patterns = Vec::with_capacity(arms.len());
+        for arm in arms {
+            if arm.pat.is_or_pat() {
+                patterns.extend(arm.pat.flatten_or_pat())
+            } else {
+                patterns.push(arm.pat)
+            }
+        }
+        Self { patterns }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.patterns.is_empty()
+    }
+    fn head_ty(&self) -> Option<Ty<'tcx>> {
+        if self.patterns.len() == 0 {
+            return None;
+        }
+        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
+        // version. Otherwise we could encounter constructors for the revealed type and crash.
+        let is_opaque = |ty: Ty<'tcx>| matches!(ty.kind(), ty::Alias(ty::Opaque, ..));
+        let first_ty = self.patterns[0].ty();
+        if is_opaque(first_ty) {
+            for pat in &self.patterns {
+                let ty = pat.ty();
+                if !is_opaque(ty) {
+                    return Some(ty);
+                }
+            }
+        }
+        Some(first_ty)
+    }
+
+    /// Do constructor splitting on the constructors of the column.
+    fn analyze_ctors(&self, pcx: &PatCtxt<'_, 'p, 'tcx>) -> SplitConstructorSet<'tcx> {
+        let column_ctors = self.patterns.iter().map(|p| p.ctor());
+        pcx.cx.ctors_for_ty(pcx.ty).split(pcx, column_ctors)
+    }
+
+    fn iter<'a>(&'a self) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
+        self.patterns.iter().copied()
+    }
+
+    /// Does specialization: given a constructor, this takes the patterns from the column that match
+    /// the constructor, and outputs their fields.
+    /// This returns one column per field of the constructor. They usually all have the same length
+    /// (the number of patterns in `self` that matched `ctor`), except that we expand or-patterns
+    /// which may change the lengths.
+    fn specialize(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>) -> Vec<Self> {
+        let arity = ctor.arity(pcx);
+        if arity == 0 {
+            return Vec::new();
+        }
+
+        // We specialize the column by `ctor`. This gives us `arity`-many columns of patterns. These
+        // columns may have different lengths in the presence of or-patterns (this is why we can't
+        // reuse `Matrix`).
+        let mut specialized_columns: Vec<_> =
+            (0..arity).map(|_| Self { patterns: Vec::new() }).collect();
+        let relevant_patterns =
+            self.patterns.iter().filter(|pat| ctor.is_covered_by(pcx, pat.ctor()));
+        for pat in relevant_patterns {
+            let specialized = pat.specialize(pcx, ctor);
+            for (subpat, column) in specialized.iter().zip(&mut specialized_columns) {
+                if subpat.is_or_pat() {
+                    column.patterns.extend(subpat.flatten_or_pat())
+                } else {
+                    column.patterns.push(subpat)
+                }
+            }
+        }
+
+        assert!(
+            !specialized_columns[0].is_empty(),
+            "ctor {ctor:?} was listed as present but isn't;
+            there is an inconsistency between `Constructor::is_covered_by` and `ConstructorSet::split`"
+        );
+        specialized_columns
+    }
+}
+
+/// Traverse the patterns to collect any variants of a non_exhaustive enum that fail to be mentioned
+/// in a given column.
+#[instrument(level = "debug", skip(cx), ret)]
+fn collect_nonexhaustive_missing_variants<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    column: &PatternColumn<'p, 'tcx>,
+) -> Vec<WitnessPat<'tcx>> {
+    let Some(ty) = column.head_ty() else {
+        return Vec::new();
+    };
+    let pcx = &PatCtxt::new_dummy(cx, ty);
+
+    let set = column.analyze_ctors(pcx);
+    if set.present.is_empty() {
+        // We can't consistently handle the case where no constructors are present (since this would
+        // require digging deep through any type in case there's a non_exhaustive enum somewhere),
+        // so for consistency we refuse to handle the top-level case, where we could handle it.
+        return vec![];
+    }
+
+    let mut witnesses = Vec::new();
+    if cx.is_foreign_non_exhaustive_enum(ty) {
+        witnesses.extend(
+            set.missing
+                .into_iter()
+                // This will list missing visible variants.
+                .filter(|c| !matches!(c, Constructor::Hidden | Constructor::NonExhaustive))
+                .map(|missing_ctor| WitnessPat::wild_from_ctor(pcx, missing_ctor)),
+        )
+    }
+
+    // Recurse into the fields.
+    for ctor in set.present {
+        let specialized_columns = column.specialize(pcx, &ctor);
+        let wild_pat = WitnessPat::wild_from_ctor(pcx, ctor);
+        for (i, col_i) in specialized_columns.iter().enumerate() {
+            // Compute witnesses for each column.
+            let wits_for_col_i = collect_nonexhaustive_missing_variants(cx, col_i);
+            // For each witness, we build a new pattern in the shape of `ctor(_, _, wit, _, _)`,
+            // adding enough wildcards to match `arity`.
+            for wit in wits_for_col_i {
+                let mut pat = wild_pat.clone();
+                pat.fields[i] = wit;
+                witnesses.push(pat);
+            }
+        }
+    }
+    witnesses
+}
+
+pub(crate) fn lint_nonexhaustive_missing_variants<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    arms: &[MatchArm<'p, 'tcx>],
+    pat_column: &PatternColumn<'p, 'tcx>,
+    scrut_ty: Ty<'tcx>,
+) {
+    if !matches!(
+        cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, cx.match_lint_level).0,
+        rustc_session::lint::Level::Allow
+    ) {
+        let witnesses = collect_nonexhaustive_missing_variants(cx, pat_column);
+        if !witnesses.is_empty() {
+            // Report that a match of a `non_exhaustive` enum marked with `non_exhaustive_omitted_patterns`
+            // is not exhaustive enough.
+            //
+            // NB: The partner lint for structs lives in `compiler/rustc_hir_analysis/src/check/pat.rs`.
+            cx.tcx.emit_spanned_lint(
+                NON_EXHAUSTIVE_OMITTED_PATTERNS,
+                cx.match_lint_level,
+                cx.scrut_span,
+                NonExhaustiveOmittedPattern {
+                    scrut_ty,
+                    uncovered: Uncovered::new(cx.scrut_span, cx, witnesses),
+                },
+            );
+        }
+    } else {
+        // We used to allow putting the `#[allow(non_exhaustive_omitted_patterns)]` on a match
+        // arm. This no longer makes sense so we warn users, to avoid silently breaking their
+        // usage of the lint.
+        for arm in arms {
+            let (lint_level, lint_level_source) =
+                cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, arm.hir_id);
+            if !matches!(lint_level, rustc_session::lint::Level::Allow) {
+                let decorator = NonExhaustiveOmittedPatternLintOnArm {
+                    lint_span: lint_level_source.span(),
+                    suggest_lint_on_match: cx.whole_match_span.map(|span| span.shrink_to_lo()),
+                    lint_level: lint_level.as_str(),
+                    lint_name: "non_exhaustive_omitted_patterns",
+                };
+
+                use rustc_errors::DecorateLint;
+                let mut err = cx.tcx.sess.struct_span_warn(arm.pat.span(), "");
+                err.set_primary_message(decorator.msg());
+                decorator.decorate_lint(&mut err);
+                err.emit();
+            }
+        }
+    }
+}
+
+/// Traverse the patterns to warn the user about ranges that overlap on their endpoints.
+#[instrument(level = "debug", skip(cx))]
+pub(crate) fn lint_overlapping_range_endpoints<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    column: &PatternColumn<'p, 'tcx>,
+) {
+    let Some(ty) = column.head_ty() else {
+        return;
+    };
+    let pcx = &PatCtxt::new_dummy(cx, ty);
+
+    let set = column.analyze_ctors(pcx);
+
+    if matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_)) {
+        let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
+            let overlap_as_pat = cx.hoist_pat_range(overlap, ty);
+            let overlaps: Vec<_> = overlapped_spans
+                .iter()
+                .copied()
+                .map(|span| Overlap { range: overlap_as_pat.clone(), span })
+                .collect();
+            cx.tcx.emit_spanned_lint(
+                lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
+                cx.match_lint_level,
+                this_span,
+                OverlappingRangeEndpoints { overlap: overlaps, range: this_span },
+            );
+        };
+
+        // If two ranges overlapped, the split set will contain their intersection as a singleton.
+        let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
+        for overlap_range in split_int_ranges.clone() {
+            if overlap_range.is_singleton() {
+                let overlap: MaybeInfiniteInt = overlap_range.lo;
+                // Ranges that look like `lo..=overlap`.
+                let mut prefixes: SmallVec<[_; 1]> = Default::default();
+                // Ranges that look like `overlap..=hi`.
+                let mut suffixes: SmallVec<[_; 1]> = Default::default();
+                // Iterate on patterns that contained `overlap`.
+                for pat in column.iter() {
+                    let this_span = pat.span();
+                    let Constructor::IntRange(this_range) = pat.ctor() else { continue };
+                    if this_range.is_singleton() {
+                        // Don't lint when one of the ranges is a singleton.
+                        continue;
+                    }
+                    if this_range.lo == overlap {
+                        // `this_range` looks like `overlap..=this_range.hi`; it overlaps with any
+                        // ranges that look like `lo..=overlap`.
+                        if !prefixes.is_empty() {
+                            emit_lint(overlap_range, this_span, &prefixes);
+                        }
+                        suffixes.push(this_span)
+                    } else if this_range.hi == overlap.plus_one() {
+                        // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
+                        // ranges that look like `overlap..=hi`.
+                        if !suffixes.is_empty() {
+                            emit_lint(overlap_range, this_span, &suffixes);
+                        }
+                        prefixes.push(this_span)
+                    }
+                }
+            }
+        }
+    } else {
+        // Recurse into the fields.
+        for ctor in set.present {
+            for col in column.specialize(pcx, &ctor) {
+                lint_overlapping_range_endpoints(cx, &col);
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
index 9cc48393fdf..353d0e4dfaf 100644
--- a/compiler/rustc_pattern_analysis/src/usefulness.rs
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -555,37 +555,29 @@ use smallvec::{smallvec, SmallVec};
 use std::fmt;
 
 use rustc_data_structures::{captures::Captures, stack::ensure_sufficient_stack};
-use rustc_hir::HirId;
 use rustc_middle::ty::{self, Ty};
-use rustc_session::lint;
-use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
 use rustc_span::{Span, DUMMY_SP};
 
-use crate::constructor::{
-    Constructor, ConstructorSet, IntRange, MaybeInfiniteInt, SplitConstructorSet,
-};
+use crate::constructor::{Constructor, ConstructorSet};
 use crate::cx::MatchCheckCtxt;
-use crate::errors::{
-    NonExhaustiveOmittedPattern, NonExhaustiveOmittedPatternLintOnArm, Overlap,
-    OverlappingRangeEndpoints, Uncovered,
-};
 use crate::pat::{DeconstructedPat, WitnessPat};
+use crate::MatchArm;
 
 use self::ValidityConstraint::*;
 
 #[derive(Copy, Clone)]
-pub(super) struct PatCtxt<'a, 'p, 'tcx> {
-    pub(super) cx: &'a MatchCheckCtxt<'p, 'tcx>,
+pub(crate) struct PatCtxt<'a, 'p, 'tcx> {
+    pub(crate) cx: &'a MatchCheckCtxt<'p, 'tcx>,
     /// Type of the current column under investigation.
-    pub(super) ty: Ty<'tcx>,
+    pub(crate) ty: Ty<'tcx>,
     /// Whether the current pattern is the whole pattern as found in a match arm, or if it's a
     /// subpattern.
-    pub(super) is_top_level: bool,
+    pub(crate) is_top_level: bool,
 }
 
 impl<'a, 'p, 'tcx> PatCtxt<'a, 'p, 'tcx> {
     /// A `PatCtxt` when code other than `is_useful` needs one.
-    fn new_dummy(cx: &'a MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
+    pub(crate) fn new_dummy(cx: &'a MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
         PatCtxt { cx, ty, is_top_level: false }
     }
 }
@@ -1279,230 +1271,6 @@ fn compute_exhaustiveness_and_usefulness<'p, 'tcx>(
     ret
 }
 
-/// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
-/// inspect the same subvalue/place".
-/// This is used to traverse patterns column-by-column for lints. Despite similarities with
-/// [`compute_exhaustiveness_and_usefulness`], this does a different traversal. Notably this is
-/// linear in the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case
-/// exponential (exhaustiveness is NP-complete). The core difference is that we treat sub-columns
-/// separately.
-///
-/// This must not contain an or-pattern. `specialize` takes care to expand them.
-///
-/// This is not used in the main algorithm; only in lints.
-#[derive(Debug)]
-struct PatternColumn<'p, 'tcx> {
-    patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>,
-}
-
-impl<'p, 'tcx> PatternColumn<'p, 'tcx> {
-    fn new(patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>) -> Self {
-        Self { patterns }
-    }
-
-    fn is_empty(&self) -> bool {
-        self.patterns.is_empty()
-    }
-    fn head_ty(&self) -> Option<Ty<'tcx>> {
-        if self.patterns.len() == 0 {
-            return None;
-        }
-        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
-        // version. Otherwise we could encounter constructors for the revealed type and crash.
-        let is_opaque = |ty: Ty<'tcx>| matches!(ty.kind(), ty::Alias(ty::Opaque, ..));
-        let first_ty = self.patterns[0].ty();
-        if is_opaque(first_ty) {
-            for pat in &self.patterns {
-                let ty = pat.ty();
-                if !is_opaque(ty) {
-                    return Some(ty);
-                }
-            }
-        }
-        Some(first_ty)
-    }
-
-    /// Do constructor splitting on the constructors of the column.
-    fn analyze_ctors(&self, pcx: &PatCtxt<'_, 'p, 'tcx>) -> SplitConstructorSet<'tcx> {
-        let column_ctors = self.patterns.iter().map(|p| p.ctor());
-        pcx.cx.ctors_for_ty(pcx.ty).split(pcx, column_ctors)
-    }
-
-    fn iter<'a>(&'a self) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.patterns.iter().copied()
-    }
-
-    /// Does specialization: given a constructor, this takes the patterns from the column that match
-    /// the constructor, and outputs their fields.
-    /// This returns one column per field of the constructor. They usually all have the same length
-    /// (the number of patterns in `self` that matched `ctor`), except that we expand or-patterns
-    /// which may change the lengths.
-    fn specialize(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>) -> Vec<Self> {
-        let arity = ctor.arity(pcx);
-        if arity == 0 {
-            return Vec::new();
-        }
-
-        // We specialize the column by `ctor`. This gives us `arity`-many columns of patterns. These
-        // columns may have different lengths in the presence of or-patterns (this is why we can't
-        // reuse `Matrix`).
-        let mut specialized_columns: Vec<_> =
-            (0..arity).map(|_| Self { patterns: Vec::new() }).collect();
-        let relevant_patterns =
-            self.patterns.iter().filter(|pat| ctor.is_covered_by(pcx, pat.ctor()));
-        for pat in relevant_patterns {
-            let specialized = pat.specialize(pcx, ctor);
-            for (subpat, column) in specialized.iter().zip(&mut specialized_columns) {
-                if subpat.is_or_pat() {
-                    column.patterns.extend(subpat.flatten_or_pat())
-                } else {
-                    column.patterns.push(subpat)
-                }
-            }
-        }
-
-        assert!(
-            !specialized_columns[0].is_empty(),
-            "ctor {ctor:?} was listed as present but isn't;
-            there is an inconsistency between `Constructor::is_covered_by` and `ConstructorSet::split`"
-        );
-        specialized_columns
-    }
-}
-
-/// Traverse the patterns to collect any variants of a non_exhaustive enum that fail to be mentioned
-/// in a given column.
-#[instrument(level = "debug", skip(cx), ret)]
-fn collect_nonexhaustive_missing_variants<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    column: &PatternColumn<'p, 'tcx>,
-) -> Vec<WitnessPat<'tcx>> {
-    let Some(ty) = column.head_ty() else {
-        return Vec::new();
-    };
-    let pcx = &PatCtxt::new_dummy(cx, ty);
-
-    let set = column.analyze_ctors(pcx);
-    if set.present.is_empty() {
-        // We can't consistently handle the case where no constructors are present (since this would
-        // require digging deep through any type in case there's a non_exhaustive enum somewhere),
-        // so for consistency we refuse to handle the top-level case, where we could handle it.
-        return vec![];
-    }
-
-    let mut witnesses = Vec::new();
-    if cx.is_foreign_non_exhaustive_enum(ty) {
-        witnesses.extend(
-            set.missing
-                .into_iter()
-                // This will list missing visible variants.
-                .filter(|c| !matches!(c, Constructor::Hidden | Constructor::NonExhaustive))
-                .map(|missing_ctor| WitnessPat::wild_from_ctor(pcx, missing_ctor)),
-        )
-    }
-
-    // Recurse into the fields.
-    for ctor in set.present {
-        let specialized_columns = column.specialize(pcx, &ctor);
-        let wild_pat = WitnessPat::wild_from_ctor(pcx, ctor);
-        for (i, col_i) in specialized_columns.iter().enumerate() {
-            // Compute witnesses for each column.
-            let wits_for_col_i = collect_nonexhaustive_missing_variants(cx, col_i);
-            // For each witness, we build a new pattern in the shape of `ctor(_, _, wit, _, _)`,
-            // adding enough wildcards to match `arity`.
-            for wit in wits_for_col_i {
-                let mut pat = wild_pat.clone();
-                pat.fields[i] = wit;
-                witnesses.push(pat);
-            }
-        }
-    }
-    witnesses
-}
-
-/// Traverse the patterns to warn the user about ranges that overlap on their endpoints.
-#[instrument(level = "debug", skip(cx))]
-fn lint_overlapping_range_endpoints<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    column: &PatternColumn<'p, 'tcx>,
-) {
-    let Some(ty) = column.head_ty() else {
-        return;
-    };
-    let pcx = &PatCtxt::new_dummy(cx, ty);
-
-    let set = column.analyze_ctors(pcx);
-
-    if matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_)) {
-        let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
-            let overlap_as_pat = cx.hoist_pat_range(overlap, ty);
-            let overlaps: Vec<_> = overlapped_spans
-                .iter()
-                .copied()
-                .map(|span| Overlap { range: overlap_as_pat.clone(), span })
-                .collect();
-            cx.tcx.emit_spanned_lint(
-                lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
-                cx.match_lint_level,
-                this_span,
-                OverlappingRangeEndpoints { overlap: overlaps, range: this_span },
-            );
-        };
-
-        // If two ranges overlapped, the split set will contain their intersection as a singleton.
-        let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
-        for overlap_range in split_int_ranges.clone() {
-            if overlap_range.is_singleton() {
-                let overlap: MaybeInfiniteInt = overlap_range.lo;
-                // Ranges that look like `lo..=overlap`.
-                let mut prefixes: SmallVec<[_; 1]> = Default::default();
-                // Ranges that look like `overlap..=hi`.
-                let mut suffixes: SmallVec<[_; 1]> = Default::default();
-                // Iterate on patterns that contained `overlap`.
-                for pat in column.iter() {
-                    let this_span = pat.span();
-                    let Constructor::IntRange(this_range) = pat.ctor() else { continue };
-                    if this_range.is_singleton() {
-                        // Don't lint when one of the ranges is a singleton.
-                        continue;
-                    }
-                    if this_range.lo == overlap {
-                        // `this_range` looks like `overlap..=this_range.hi`; it overlaps with any
-                        // ranges that look like `lo..=overlap`.
-                        if !prefixes.is_empty() {
-                            emit_lint(overlap_range, this_span, &prefixes);
-                        }
-                        suffixes.push(this_span)
-                    } else if this_range.hi == overlap.plus_one() {
-                        // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
-                        // ranges that look like `overlap..=hi`.
-                        if !suffixes.is_empty() {
-                            emit_lint(overlap_range, this_span, &suffixes);
-                        }
-                        prefixes.push(this_span)
-                    }
-                }
-            }
-        }
-    } else {
-        // Recurse into the fields.
-        for ctor in set.present {
-            for col in column.specialize(pcx, &ctor) {
-                lint_overlapping_range_endpoints(cx, &col);
-            }
-        }
-    }
-}
-
-/// The arm of a match expression.
-#[derive(Clone, Copy, Debug)]
-pub struct MatchArm<'p, 'tcx> {
-    /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`.
-    pub pat: &'p DeconstructedPat<'p, 'tcx>,
-    pub hir_id: HirId,
-    pub has_guard: bool,
-}
-
 /// Indicates whether or not a given arm is useful.
 #[derive(Clone, Debug)]
 pub enum Usefulness {
@@ -1524,10 +1292,9 @@ pub struct UsefulnessReport<'p, 'tcx> {
     pub non_exhaustiveness_witnesses: Vec<WitnessPat<'tcx>>,
 }
 
-/// The entrypoint for this file. Computes whether a match is exhaustive and which of its arms are
-/// useful.
+/// Computes whether a match is exhaustive and which of its arms are useful.
 #[instrument(skip(cx, arms), level = "debug")]
-pub fn compute_match_usefulness<'p, 'tcx>(
+pub(crate) fn compute_match_usefulness<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
     arms: &[MatchArm<'p, 'tcx>],
     scrut_ty: Ty<'tcx>,
@@ -1551,59 +1318,5 @@ pub fn compute_match_usefulness<'p, 'tcx>(
             (arm, usefulness)
         })
         .collect();
-    let report = UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses };
-
-    let pat_column = PatternColumn::new(matrix.heads().collect());
-    // Lint on ranges that overlap on their endpoints, which is likely a mistake.
-    lint_overlapping_range_endpoints(cx, &pat_column);
-
-    // Run the non_exhaustive_omitted_patterns lint. Only run on refutable patterns to avoid hitting
-    // `if let`s. Only run if the match is exhaustive otherwise the error is redundant.
-    if cx.refutable && report.non_exhaustiveness_witnesses.is_empty() {
-        if !matches!(
-            cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, cx.match_lint_level).0,
-            rustc_session::lint::Level::Allow
-        ) {
-            let witnesses = collect_nonexhaustive_missing_variants(cx, &pat_column);
-            if !witnesses.is_empty() {
-                // Report that a match of a `non_exhaustive` enum marked with `non_exhaustive_omitted_patterns`
-                // is not exhaustive enough.
-                //
-                // NB: The partner lint for structs lives in `compiler/rustc_hir_analysis/src/check/pat.rs`.
-                cx.tcx.emit_spanned_lint(
-                    NON_EXHAUSTIVE_OMITTED_PATTERNS,
-                    cx.match_lint_level,
-                    cx.scrut_span,
-                    NonExhaustiveOmittedPattern {
-                        scrut_ty,
-                        uncovered: Uncovered::new(cx.scrut_span, cx, witnesses),
-                    },
-                );
-            }
-        } else {
-            // We used to allow putting the `#[allow(non_exhaustive_omitted_patterns)]` on a match
-            // arm. This no longer makes sense so we warn users, to avoid silently breaking their
-            // usage of the lint.
-            for arm in arms {
-                let (lint_level, lint_level_source) =
-                    cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, arm.hir_id);
-                if !matches!(lint_level, rustc_session::lint::Level::Allow) {
-                    let decorator = NonExhaustiveOmittedPatternLintOnArm {
-                        lint_span: lint_level_source.span(),
-                        suggest_lint_on_match: cx.whole_match_span.map(|span| span.shrink_to_lo()),
-                        lint_level: lint_level.as_str(),
-                        lint_name: "non_exhaustive_omitted_patterns",
-                    };
-
-                    use rustc_errors::DecorateLint;
-                    let mut err = cx.tcx.sess.struct_span_warn(arm.pat.span(), "");
-                    err.set_primary_message(decorator.msg());
-                    decorator.decorate_lint(&mut err);
-                    err.emit();
-                }
-            }
-        }
-    }
-
-    report
+    UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses }
 }
-- 
cgit 1.4.1-3-g733a5


From de3f983bcdafcf9ba1bd322661f85e3ab1d50e15 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 11 Dec 2023 11:10:19 +0100
Subject: Make `MaybeInfiniteInt` rustc-independent

---
 compiler/rustc_pattern_analysis/src/constructor.rs | 44 ++++++++++---------
 compiler/rustc_pattern_analysis/src/cx.rs          | 49 +++++++++++++++-------
 2 files changed, 59 insertions(+), 34 deletions(-)

diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index aa02ca27add..16438ae574d 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -159,9 +159,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::RangeEnd;
 use rustc_index::IndexVec;
 use rustc_middle::mir::Const;
-use rustc_middle::ty::layout::IntegerExt;
-use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_target::abi::{Integer, VariantIdx};
+use rustc_target::abi::VariantIdx;
 
 use self::Constructor::*;
 use self::MaybeInfiniteInt::*;
@@ -183,6 +181,7 @@ enum Presence {
 pub enum MaybeInfiniteInt {
     NegInfinity,
     /// Encoded value. DO NOT CONSTRUCT BY HAND; use `new_finite`.
+    #[non_exhaustive]
     Finite(u128),
     /// The integer after `u128::MAX`. We need it to represent `x..=u128::MAX` as an exclusive range.
     JustAfterMax,
@@ -190,23 +189,31 @@ pub enum MaybeInfiniteInt {
 }
 
 impl MaybeInfiniteInt {
-    // The return value of `signed_bias` should be XORed with a value to encode/decode it.
-    pub(crate) fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
-        match *ty.kind() {
-            ty::Int(ity) => {
-                let bits = Integer::from_int_ty(&tcx, ity).size().bits() as u128;
-                1u128 << (bits - 1)
-            }
-            _ => 0,
-        }
+    pub fn new_finite_uint(bits: u128) -> Self {
+        Finite(bits)
     }
-
-    pub fn new_finite(tcx: TyCtxt<'_>, ty: Ty<'_>, bits: u128) -> Self {
-        let bias = Self::signed_bias(tcx, ty);
+    pub fn new_finite_int(bits: u128, size: u64) -> Self {
         // Perform a shift if the underlying types are signed, which makes the interval arithmetic
         // type-independent.
-        let x = bits ^ bias;
-        Finite(x)
+        let bias = 1u128 << (size - 1);
+        Finite(bits ^ bias)
+    }
+
+    pub fn as_finite_uint(self) -> Option<u128> {
+        match self {
+            Finite(bits) => Some(bits),
+            _ => None,
+        }
+    }
+    pub fn as_finite_int(self, size: u64) -> Option<u128> {
+        // We decode the shift.
+        match self {
+            Finite(bits) => {
+                let bias = 1u128 << (size - 1);
+                Some(bits ^ bias)
+            }
+            _ => None,
+        }
     }
 
     /// Note: this will not turn a finite value into an infinite one or vice-versa.
@@ -253,8 +260,7 @@ impl IntRange {
     }
 
     #[inline]
-    pub fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
-        let x = MaybeInfiniteInt::new_finite(tcx, ty, bits);
+    pub fn from_singleton(x: MaybeInfiniteInt) -> IntRange {
         IntRange { lo: x, hi: x.plus_one() }
     }
 
diff --git a/compiler/rustc_pattern_analysis/src/cx.rs b/compiler/rustc_pattern_analysis/src/cx.rs
index ffcbd2f5d83..8a4f39a1f4a 100644
--- a/compiler/rustc_pattern_analysis/src/cx.rs
+++ b/compiler/rustc_pattern_analysis/src/cx.rs
@@ -204,10 +204,10 @@ impl<'p, 'tcx> MatchCheckCtxt<'p, 'tcx> {
     #[instrument(level = "debug", skip(self), ret)]
     pub fn ctors_for_ty(&self, ty: Ty<'tcx>) -> ConstructorSet {
         let cx = self;
-        let make_range = |start, end| {
+        let make_uint_range = |start, end| {
             IntRange::from_range(
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, start),
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, end),
+                MaybeInfiniteInt::new_finite_uint(start),
+                MaybeInfiniteInt::new_finite_uint(end),
                 RangeEnd::Included,
             )
         };
@@ -218,8 +218,8 @@ impl<'p, 'tcx> MatchCheckCtxt<'p, 'tcx> {
             ty::Char => {
                 // The valid Unicode Scalar Value ranges.
                 ConstructorSet::Integers {
-                    range_1: make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
-                    range_2: Some(make_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
+                    range_1: make_uint_range('\u{0000}' as u128, '\u{D7FF}' as u128),
+                    range_2: Some(make_uint_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
                 }
             }
             &ty::Int(ity) => {
@@ -230,22 +230,24 @@ impl<'p, 'tcx> MatchCheckCtxt<'p, 'tcx> {
                         hi: MaybeInfiniteInt::PosInfinity,
                     }
                 } else {
-                    let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
-                    let min = 1u128 << (bits - 1);
+                    let size = Integer::from_int_ty(&cx.tcx, ity).size().bits();
+                    let min = 1u128 << (size - 1);
                     let max = min - 1;
-                    make_range(min, max)
+                    let min = MaybeInfiniteInt::new_finite_int(min, size);
+                    let max = MaybeInfiniteInt::new_finite_int(max, size);
+                    IntRange::from_range(min, max, RangeEnd::Included)
                 };
                 ConstructorSet::Integers { range_1: range, range_2: None }
             }
             &ty::Uint(uty) => {
                 let range = if ty.is_ptr_sized_integral() {
                     // The max value of `usize` is not allowed to be observed.
-                    let lo = MaybeInfiniteInt::new_finite(cx.tcx, ty, 0);
+                    let lo = MaybeInfiniteInt::new_finite_uint(0);
                     IntRange { lo, hi: MaybeInfiniteInt::PosInfinity }
                 } else {
                     let size = Integer::from_uint_ty(&cx.tcx, uty).size();
                     let max = size.truncate(u128::MAX);
-                    make_range(0, max)
+                    make_uint_range(0, max)
                 };
                 ConstructorSet::Integers { range_1: range, range_2: None }
             }
@@ -329,7 +331,13 @@ impl<'p, 'tcx> MatchCheckCtxt<'p, 'tcx> {
             PatRangeBoundary::NegInfinity => MaybeInfiniteInt::NegInfinity,
             PatRangeBoundary::Finite(value) => {
                 let bits = value.eval_bits(self.tcx, self.param_env);
-                MaybeInfiniteInt::new_finite(self.tcx, ty, bits)
+                match *ty.kind() {
+                    ty::Int(ity) => {
+                        let size = Integer::from_int_ty(&self.tcx, ity).size().bits();
+                        MaybeInfiniteInt::new_finite_int(bits, size)
+                    }
+                    _ => MaybeInfiniteInt::new_finite_uint(bits),
+                }
             }
             PatRangeBoundary::PosInfinity => MaybeInfiniteInt::PosInfinity,
         }
@@ -428,7 +436,16 @@ impl<'p, 'tcx> MatchCheckCtxt<'p, 'tcx> {
                     }
                     ty::Char | ty::Int(_) | ty::Uint(_) => {
                         ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => IntRange(IntRange::from_bits(cx.tcx, pat.ty, bits)),
+                            Some(bits) => {
+                                let x = match *pat.ty.kind() {
+                                    ty::Int(ity) => {
+                                        let size = Integer::from_int_ty(&cx.tcx, ity).size().bits();
+                                        MaybeInfiniteInt::new_finite_int(bits, size)
+                                    }
+                                    _ => MaybeInfiniteInt::new_finite_uint(bits),
+                                };
+                                IntRange(IntRange::from_singleton(x))
+                            }
                             None => Opaque(OpaqueId::new()),
                         };
                         fields = &[];
@@ -559,10 +576,12 @@ impl<'p, 'tcx> MatchCheckCtxt<'p, 'tcx> {
         let tcx = self.tcx;
         match miint {
             NegInfinity => PatRangeBoundary::NegInfinity,
-            Finite(x) => {
-                let bias = MaybeInfiniteInt::signed_bias(tcx, ty);
-                let bits = x ^ bias;
+            Finite(_) => {
                 let size = ty.primitive_size(tcx);
+                let bits = match *ty.kind() {
+                    ty::Int(_) => miint.as_finite_int(size.bits()).unwrap(),
+                    _ => miint.as_finite_uint().unwrap(),
+                };
                 match Scalar::try_from_uint(bits, size) {
                     Some(scalar) => {
                         let value = mir::Const::from_scalar(tcx, scalar, ty);
-- 
cgit 1.4.1-3-g733a5


From 5d6c539c2d89a184fc9f03e3783cbf2b8af0e29b Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 11 Dec 2023 10:56:21 +0100
Subject: Fix item visibilities

---
 compiler/rustc_pattern_analysis/src/constructor.rs | 18 +++++++++---------
 compiler/rustc_pattern_analysis/src/pat.rs         | 22 +++++++++++-----------
 compiler/rustc_pattern_analysis/src/usefulness.rs  | 18 +++++++-----------
 3 files changed, 27 insertions(+), 31 deletions(-)

diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index 16438ae574d..e9e873f14b9 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -253,7 +253,7 @@ pub struct IntRange {
 
 impl IntRange {
     /// Best effort; will not know that e.g. `255u8..` is a singleton.
-    pub fn is_singleton(&self) -> bool {
+    pub(crate) fn is_singleton(&self) -> bool {
         // Since `lo` and `hi` can't be the same `Infinity` and `plus_one` never changes from finite
         // to infinite, this correctly only detects ranges that contain exacly one `Finite(x)`.
         self.lo.plus_one() == self.hi
@@ -670,11 +670,11 @@ pub enum Constructor<'tcx> {
 }
 
 impl<'tcx> Constructor<'tcx> {
-    pub(super) fn is_non_exhaustive(&self) -> bool {
+    pub(crate) fn is_non_exhaustive(&self) -> bool {
         matches!(self, NonExhaustive)
     }
 
-    pub(super) fn as_variant(&self) -> Option<VariantIdx> {
+    pub(crate) fn as_variant(&self) -> Option<VariantIdx> {
         match self {
             Variant(i) => Some(*i),
             _ => None,
@@ -686,7 +686,7 @@ impl<'tcx> Constructor<'tcx> {
             _ => None,
         }
     }
-    pub(super) fn as_int_range(&self) -> Option<&IntRange> {
+    pub(crate) fn as_int_range(&self) -> Option<&IntRange> {
         match self {
             IntRange(range) => Some(range),
             _ => None,
@@ -830,10 +830,10 @@ pub enum ConstructorSet {
 /// of the `ConstructorSet` for the type, yet if we forgot to include them in `present` we would be
 /// ignoring any row with `Opaque`s in the algorithm. Hence the importance of point 4.
 #[derive(Debug)]
-pub(super) struct SplitConstructorSet<'tcx> {
-    pub(super) present: SmallVec<[Constructor<'tcx>; 1]>,
-    pub(super) missing: Vec<Constructor<'tcx>>,
-    pub(super) missing_empty: Vec<Constructor<'tcx>>,
+pub(crate) struct SplitConstructorSet<'tcx> {
+    pub(crate) present: SmallVec<[Constructor<'tcx>; 1]>,
+    pub(crate) missing: Vec<Constructor<'tcx>>,
+    pub(crate) missing_empty: Vec<Constructor<'tcx>>,
 }
 
 impl ConstructorSet {
@@ -842,7 +842,7 @@ impl ConstructorSet {
     /// or slices. This can get subtle; see [`SplitConstructorSet`] for details of this operation
     /// and its invariants.
     #[instrument(level = "debug", skip(self, pcx, ctors), ret)]
-    pub(super) fn split<'a, 'tcx>(
+    pub(crate) fn split<'a, 'tcx>(
         &self,
         pcx: &PatCtxt<'_, '_, 'tcx>,
         ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
index bdbe14a08e3..404651124ad 100644
--- a/compiler/rustc_pattern_analysis/src/pat.rs
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -37,11 +37,11 @@ pub struct DeconstructedPat<'p, 'tcx> {
 }
 
 impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
-    pub(super) fn wildcard(ty: Ty<'tcx>, span: Span) -> Self {
+    pub fn wildcard(ty: Ty<'tcx>, span: Span) -> Self {
         Self::new(Wildcard, &[], ty, span)
     }
 
-    pub(super) fn new(
+    pub fn new(
         ctor: Constructor<'tcx>,
         fields: &'p [DeconstructedPat<'p, 'tcx>],
         ty: Ty<'tcx>,
@@ -50,11 +50,11 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
         DeconstructedPat { ctor, fields, ty, span, useful: Cell::new(false) }
     }
 
-    pub(super) fn is_or_pat(&self) -> bool {
+    pub(crate) fn is_or_pat(&self) -> bool {
         matches!(self.ctor, Or)
     }
     /// Expand this (possibly-nested) or-pattern into its alternatives.
-    pub(super) fn flatten_or_pat(&'p self) -> SmallVec<[&'p Self; 1]> {
+    pub(crate) fn flatten_or_pat(&'p self) -> SmallVec<[&'p Self; 1]> {
         if self.is_or_pat() {
             self.iter_fields().flat_map(|p| p.flatten_or_pat()).collect()
         } else {
@@ -80,7 +80,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
 
     /// Specialize this pattern with a constructor.
     /// `other_ctor` can be different from `self.ctor`, but must be covered by it.
-    pub(super) fn specialize<'a>(
+    pub(crate) fn specialize<'a>(
         &'a self,
         pcx: &PatCtxt<'_, 'p, 'tcx>,
         other_ctor: &Constructor<'tcx>,
@@ -122,10 +122,10 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
 
     /// We keep track for each pattern if it was ever useful during the analysis. This is used
     /// with `redundant_spans` to report redundant subpatterns arising from or patterns.
-    pub(super) fn set_useful(&self) {
+    pub(crate) fn set_useful(&self) {
         self.useful.set(true)
     }
-    pub(super) fn is_useful(&self) -> bool {
+    pub(crate) fn is_useful(&self) -> bool {
         if self.useful.get() {
             true
         } else if self.is_or_pat() && self.iter_fields().any(|f| f.is_useful()) {
@@ -140,7 +140,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
     }
 
     /// Report the spans of subpatterns that were not useful, if any.
-    pub(super) fn redundant_spans(&self) -> Vec<Span> {
+    pub(crate) fn redundant_spans(&self) -> Vec<Span> {
         let mut spans = Vec::new();
         self.collect_redundant_spans(&mut spans);
         spans
@@ -175,17 +175,17 @@ pub struct WitnessPat<'tcx> {
 }
 
 impl<'tcx> WitnessPat<'tcx> {
-    pub(super) fn new(ctor: Constructor<'tcx>, fields: Vec<Self>, ty: Ty<'tcx>) -> Self {
+    pub(crate) fn new(ctor: Constructor<'tcx>, fields: Vec<Self>, ty: Ty<'tcx>) -> Self {
         Self { ctor, fields, ty }
     }
-    pub(super) fn wildcard(ty: Ty<'tcx>) -> Self {
+    pub(crate) fn wildcard(ty: Ty<'tcx>) -> Self {
         Self::new(Wildcard, Vec::new(), ty)
     }
 
     /// Construct a pattern that matches everything that starts with this constructor.
     /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
     /// `Some(_)`.
-    pub(super) fn wild_from_ctor(pcx: &PatCtxt<'_, '_, 'tcx>, ctor: Constructor<'tcx>) -> Self {
+    pub(crate) fn wild_from_ctor(pcx: &PatCtxt<'_, '_, 'tcx>, ctor: Constructor<'tcx>) -> Self {
         let field_tys =
             pcx.cx.ctor_wildcard_fields(&ctor, pcx.ty).iter().map(|deco_pat| deco_pat.ty());
         let fields = field_tys.map(|ty| Self::wildcard(ty)).collect();
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
index 353d0e4dfaf..1c3de8803d3 100644
--- a/compiler/rustc_pattern_analysis/src/usefulness.rs
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -594,7 +594,7 @@ impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
 /// - in the matrix, track whether a given place (aka column) is known to contain a valid value or
 ///     not.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub(super) enum ValidityConstraint {
+enum ValidityConstraint {
     ValidOnly,
     MaybeInvalid,
     /// Option for backwards compatibility: the place is not known to be valid but we allow omitting
@@ -603,7 +603,7 @@ pub(super) enum ValidityConstraint {
 }
 
 impl ValidityConstraint {
-    pub(super) fn from_bool(is_valid_only: bool) -> Self {
+    fn from_bool(is_valid_only: bool) -> Self {
         if is_valid_only { ValidOnly } else { MaybeInvalid }
     }
 
@@ -615,10 +615,10 @@ impl ValidityConstraint {
         }
     }
 
-    pub(super) fn is_known_valid(self) -> bool {
+    fn is_known_valid(self) -> bool {
         matches!(self, ValidOnly)
     }
-    pub(super) fn allows_omitting_empty_arms(self) -> bool {
+    fn allows_omitting_empty_arms(self) -> bool {
         matches!(self, ValidOnly | MaybeInvalidButAllowOmittingArms)
     }
 
@@ -628,11 +628,7 @@ impl ValidityConstraint {
     ///
     /// Pending further opsem decisions, the current behavior is: validity is preserved, except
     /// inside `&` and union fields where validity is reset to `MaybeInvalid`.
-    pub(super) fn specialize<'tcx>(
-        self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        ctor: &Constructor<'tcx>,
-    ) -> Self {
+    fn specialize<'tcx>(self, pcx: &PatCtxt<'_, '_, 'tcx>, ctor: &Constructor<'tcx>) -> Self {
         // We preserve validity except when we go inside a reference or a union field.
         if matches!(ctor, Constructor::Single)
             && (matches!(pcx.ty.kind(), ty::Ref(..))
@@ -1023,7 +1019,7 @@ impl<'p, 'tcx> fmt::Debug for Matrix<'p, 'tcx> {
 ///
 /// See the top of the file for more detailed explanations and examples.
 #[derive(Debug, Clone)]
-pub(crate) struct WitnessStack<'tcx>(Vec<WitnessPat<'tcx>>);
+struct WitnessStack<'tcx>(Vec<WitnessPat<'tcx>>);
 
 impl<'tcx> WitnessStack<'tcx> {
     /// Asserts that the witness contains a single pattern, and returns it.
@@ -1070,7 +1066,7 @@ impl<'tcx> WitnessStack<'tcx> {
 /// Just as the `Matrix` starts with a single column, by the end of the algorithm, this has a single
 /// column, which contains the patterns that are missing for the match to be exhaustive.
 #[derive(Debug, Clone)]
-pub struct WitnessMatrix<'tcx>(Vec<WitnessStack<'tcx>>);
+struct WitnessMatrix<'tcx>(Vec<WitnessStack<'tcx>>);
 
 impl<'tcx> WitnessMatrix<'tcx> {
     /// New matrix with no witnesses.
-- 
cgit 1.4.1-3-g733a5


From 79806f1e46c758b02fd473c5a0b70f43338f0a08 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 11 Dec 2023 11:02:24 +0100
Subject: Include the new crate in triagebot pings

---
 triagebot.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/triagebot.toml b/triagebot.toml
index 775d9714c60..2d9a5dabc8a 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -480,6 +480,10 @@ cc = ["@GuillaumeGomez"]
 message = "Some changes might have occurred in exhaustiveness checking"
 cc = ["@Nadrieril"]
 
+[mentions."compiler/rustc_pattern_analysis"]
+message = "Some changes might have occurred in exhaustiveness checking"
+cc = ["@Nadrieril"]
+
 [mentions."library/portable-simd"]
 message = """
 Portable SIMD is developed in its own repository. If possible, consider \
-- 
cgit 1.4.1-3-g733a5


From 43714edb6f291892f571875b42235208f075884f Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 11 Dec 2023 12:53:01 +0100
Subject: Fix doc links

---
 compiler/rustc_pattern_analysis/src/constructor.rs | 24 +++++++++++-----------
 compiler/rustc_pattern_analysis/src/lints.rs       |  9 ++++----
 compiler/rustc_pattern_analysis/src/usefulness.rs  | 11 +++++-----
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index e9e873f14b9..716ccdd4dcd 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -1,18 +1,18 @@
-//! As explained in [`super::usefulness`], values and patterns are made from constructors applied to
+//! As explained in [`crate::usefulness`], values and patterns are made from constructors applied to
 //! fields. This file defines a `Constructor` enum and various operations to manipulate them.
 //!
 //! There are two important bits of core logic in this file: constructor inclusion and constructor
 //! splitting. Constructor inclusion, i.e. whether a constructor is included in/covered by another,
 //! is straightforward and defined in [`Constructor::is_covered_by`].
 //!
-//! Constructor splitting is mentioned in [`super::usefulness`] but not detailed. We describe it
+//! Constructor splitting is mentioned in [`crate::usefulness`] but not detailed. We describe it
 //! precisely here.
 //!
 //!
 //!
 //! # Constructor grouping and splitting
 //!
-//! As explained in the corresponding section in [`super::usefulness`], to make usefulness tractable
+//! As explained in the corresponding section in [`crate::usefulness`], to make usefulness tractable
 //! we need to group together constructors that have the same effect when they are used to
 //! specialize the matrix.
 //!
@@ -28,7 +28,7 @@
 //! In this example we can restrict specialization to 5 cases: `0..50`, `50..=100`, `101..=150`,
 //! `151..=200` and `200..`.
 //!
-//! In [`super::usefulness`], we had said that `specialize` only takes value-only constructors. We
+//! In [`crate::usefulness`], we had said that `specialize` only takes value-only constructors. We
 //! now relax this restriction: we allow `specialize` to take constructors like `0..50` as long as
 //! we're careful to only do that with constructors that make sense. For example, `specialize(0..50,
 //! (0..=100, true))` is sensible, but `specialize(50..=200, (0..=100, true))` is not.
@@ -40,9 +40,9 @@
 //! - That have no non-trivial intersection with any of the constructors in the column (i.e. they're
 //!     each either disjoint with or covered by any given column constructor).
 //!
-//! We compute this in two steps: first [`ConstructorSet::for_ty`] determines the set of all
-//! possible constructors for the type. Then [`ConstructorSet::split`] looks at the column of
-//! constructors and splits the set into groups accordingly. The precise invariants of
+//! We compute this in two steps: first [`crate::cx::MatchCheckCtxt::ctors_for_ty`] determines the
+//! set of all possible constructors for the type. Then [`ConstructorSet::split`] looks at the
+//! column of constructors and splits the set into groups accordingly. The precise invariants of
 //! [`ConstructorSet::split`] is described in [`SplitConstructorSet`].
 //!
 //! Constructor splitting has two interesting special cases: integer range splitting (see
@@ -71,10 +71,10 @@
 //! `Wildcard`.
 //!
 //! The only place where we care about which constructors `Missing` represents is in diagnostics
-//! (see `super::usefulness::WitnessMatrix::apply_constructor`).
+//! (see `crate::usefulness::WitnessMatrix::apply_constructor`).
 //!
 //! We choose whether to specialize with `Missing` in
-//! `super::usefulness::compute_exhaustiveness_and_reachability`.
+//! `crate::usefulness::compute_exhaustiveness_and_usefulness`.
 //!
 //!
 //!
@@ -88,7 +88,7 @@
 //! `exhaustive_patterns` feature is turned on, in which case we do treat them as empty. And also
 //! except if the type has no constructors (like `enum Void {}` but not like `Result<!, !>`), we
 //! specifically allow `match void {}` to be exhaustive. There are additionally considerations of
-//! place validity that are handled in `super::usefulness`. Yes this is a bit tricky.
+//! place validity that are handled in `crate::usefulness`. Yes this is a bit tricky.
 //!
 //! The second thing is that regardless of the above, it is always allowed to use all the
 //! constructors of a type. For example, all the following is ok:
@@ -136,8 +136,8 @@
 //! the algorithm can't distinguish them from a nonempty constructor. The only known case where this
 //! could happen is the `[..]` pattern on `[!; N]` with `N > 0` so we must take care to not emit it.
 //!
-//! This is all handled by [`ConstructorSet::for_ty`] and [`ConstructorSet::split`]. The invariants
-//! of [`SplitConstructorSet`] are also of interest.
+//! This is all handled by [`crate::cx::MatchCheckCtxt::ctors_for_ty`] and
+//! [`ConstructorSet::split`]. The invariants of [`SplitConstructorSet`] are also of interest.
 //!
 //!
 //!
diff --git a/compiler/rustc_pattern_analysis/src/lints.rs b/compiler/rustc_pattern_analysis/src/lints.rs
index 130945870e4..8ab559c9e7a 100644
--- a/compiler/rustc_pattern_analysis/src/lints.rs
+++ b/compiler/rustc_pattern_analysis/src/lints.rs
@@ -18,11 +18,10 @@ use crate::MatchArm;
 
 /// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
 /// inspect the same subvalue/place".
-/// This is used to traverse patterns column-by-column for lints. Despite similarities with
-/// [`compute_exhaustiveness_and_usefulness`], this does a different traversal. Notably this is
-/// linear in the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case
-/// exponential (exhaustiveness is NP-complete). The core difference is that we treat sub-columns
-/// separately.
+/// This is used to traverse patterns column-by-column for lints. Despite similarities with the
+/// algorithm in [`crate::usefulness`], this does a different traversal. Notably this is linear in
+/// the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case exponential
+/// (exhaustiveness is NP-complete). The core difference is that we treat sub-columns separately.
 ///
 /// This must not contain an or-pattern. `specialize` takes care to expand them.
 ///
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
index 1c3de8803d3..f268a551547 100644
--- a/compiler/rustc_pattern_analysis/src/usefulness.rs
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -97,8 +97,9 @@
 //! - `matches!([v0], [p0, .., p1]) := false` (incompatible lengths)
 //! - `matches!([v0, v1, v2], [p0, .., p1]) := matches!(v0, p0) && matches!(v2, p1)`
 //!
-//! Constructors, fields and relevant operations are defined in the [`super::deconstruct_pat`]
-//! module. The question of whether a constructor is matched by another one is answered by
+//! Constructors and relevant operations are defined in the [`crate::constructor`] module. A
+//! representation of patterns that uses constructors is available in [`crate::pat`]. The question
+//! of whether a constructor is matched by another one is answered by
 //! [`Constructor::is_covered_by`].
 //!
 //! Note 1: variable bindings (like the `x` in `Some(x)`) match anything, so we treat them as wildcards.
@@ -241,8 +242,8 @@
 //! Therefore `usefulness(tp_1, tp_2, tq)` returns the single witness-tuple `[Variant2(Some(true), 0)]`.
 //!
 //!
-//! Computing the set of constructors for a type is done in [`ConstructorSet::for_ty`]. See the
-//! following sections for more accurate versions of the algorithm and corresponding links.
+//! Computing the set of constructors for a type is done in [`MatchCheckCtxt::ctors_for_ty`]. See
+//! the following sections for more accurate versions of the algorithm and corresponding links.
 //!
 //!
 //!
@@ -295,7 +296,7 @@
 //! the same reasoning, we only need to try two cases: `North`, and "everything else".
 //!
 //! We call _constructor splitting_ the operation that computes such a minimal set of cases to try.
-//! This is done in [`ConstructorSet::split`] and explained in [`super::deconstruct_pat`].
+//! This is done in [`ConstructorSet::split`] and explained in [`crate::constructor`].
 //!
 //!
 //!
-- 
cgit 1.4.1-3-g733a5