Auto merge of #85273 - LeSeulArtichaut:thir-query, r=nikomatsakis

Make building THIR a stealable query

This PR creates a stealable `thir_body` query so that we can build the THIR only once for THIR unsafeck and MIR build.

Blocked on #83842.
r? `@nikomatsakis`

42 files changed, 937 insertions, 922 deletions

diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index e2220e3b60d..70ffff1ab99 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -873,9 +873,8 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
 
     sess.time("MIR_effect_checking", || {
         for def_id in tcx.body_owners() {
-            if tcx.sess.opts.debugging_opts.thir_unsafeck {
-                tcx.ensure().thir_check_unsafety(def_id);
-            } else {
+            tcx.ensure().thir_check_unsafety(def_id);
+            if !tcx.sess.opts.debugging_opts.thir_unsafeck {
                 mir::transform::check_unsafety::check_unsafety(tcx, def_id);
             }
 
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index d5697513eef..a89d00e26ac 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -14,6 +14,7 @@ macro_rules! arena_types {
             [] layouts: rustc_target::abi::Layout,
             // AdtDef are interned and compared by address
             [] adt_def: rustc_middle::ty::AdtDef,
+            [] steal_thir: rustc_data_structures::steal::Steal<rustc_middle::thir::Thir<$tcx>>,
             [] steal_mir: rustc_data_structures::steal::Steal<rustc_middle::mir::Body<$tcx>>,
             [decode] mir: rustc_middle::mir::Body<$tcx>,
             [] steal_promoted:
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index aa54d1ae7b9..8476929eaec 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -285,7 +285,7 @@ pub type DepNode = rustc_query_system::dep_graph::DepNode<DepKind>;
 // required that their size stay the same, but we don't want to change
 // it inadvertently. This assert just ensures we're aware of any change.
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-static_assert_size!(DepNode, 17);
+static_assert_size!(DepNode, 18);
 
 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
 static_assert_size!(DepNode, 24);
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 597a4fd0f52..04124ca0281 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -83,6 +83,7 @@ pub mod infer;
 pub mod lint;
 pub mod middle;
 pub mod mir;
+pub mod thir;
 pub mod traits;
 pub mod ty;
 
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 70f70788bca..9125be33c93 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -220,6 +220,11 @@ rustc_queries! {
         desc { "checking if the crate is_panic_runtime" }
     }
 
+    /// Fetch the THIR for a given body. If typeck for that body failed, returns an empty `Thir`.
+    query thir_body(key: ty::WithOptConstParam<LocalDefId>) -> (&'tcx Steal<thir::Thir<'tcx>>, thir::ExprId) {
+        desc { |tcx| "building THIR for `{}`", tcx.def_path_str(key.did.to_def_id()) }
+    }
+
     /// Set of all the `DefId`s in this crate that have MIR associated with
     /// them. This includes all the body owners, but also things like struct
     /// constructors.
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
new file mode 100644
index 00000000000..a5069113702
--- /dev/null
+++ b/compiler/rustc_middle/src/thir.rs
@@ -0,0 +1,747 @@
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_hir as hir;
+use rustc_hir::def::CtorKind;
+use rustc_hir::def_id::DefId;
+use rustc_hir::RangeEnd;
+use rustc_index::newtype_index;
+use rustc_index::vec::{Idx, IndexVec};
+use rustc_middle::infer::canonical::Canonical;
+use rustc_middle::middle::region;
+use rustc_middle::mir::{
+    BinOp, BorrowKind, FakeReadCause, Field, Mutability, UnOp, UserTypeProjection,
+};
+use rustc_middle::ty::adjustment::PointerCast;
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_middle::ty::{self, AdtDef, Const, Ty, UpvarSubsts, UserType};
+use rustc_middle::ty::{
+    CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations,
+};
+use rustc_span::{Span, Symbol, DUMMY_SP};
+use rustc_target::abi::VariantIdx;
+use rustc_target::asm::InlineAsmRegOrRegClass;
+
+use std::fmt;
+use std::ops::Index;
+
+newtype_index! {
+    #[derive(HashStable)]
+    pub struct ArmId {
+        DEBUG_FORMAT = "a{}"
+    }
+}
+
+newtype_index! {
+    #[derive(HashStable)]
+    pub struct ExprId {
+        DEBUG_FORMAT = "e{}"
+    }
+}
+
+newtype_index! {
+    #[derive(HashStable)]
+    pub struct StmtId {
+        DEBUG_FORMAT = "s{}"
+    }
+}
+
+macro_rules! thir_with_elements {
+    ($($name:ident: $id:ty => $value:ty,)*) => {
+        #[derive(Debug, HashStable)]
+        pub struct Thir<'tcx> {
+            $(
+                pub $name: IndexVec<$id, $value>,
+            )*
+        }
+
+        impl<'tcx> Thir<'tcx> {
+            pub fn new() -> Thir<'tcx> {
+                Thir {
+                    $(
+                        $name: IndexVec::new(),
+                    )*
+                }
+            }
+        }
+
+        $(
+            impl<'tcx> Index<$id> for Thir<'tcx> {
+                type Output = $value;
+                fn index(&self, index: $id) -> &Self::Output {
+                    &self.$name[index]
+                }
+            }
+        )*
+    }
+}
+
+thir_with_elements! {
+    arms: ArmId => Arm<'tcx>,
+    exprs: ExprId => Expr<'tcx>,
+    stmts: StmtId => Stmt<'tcx>,
+}
+
+#[derive(Copy, Clone, Debug, HashStable)]
+pub enum LintLevel {
+    Inherited,
+    Explicit(hir::HirId),
+}
+
+#[derive(Debug, HashStable)]
+pub struct Block {
+    pub targeted_by_break: bool,
+    pub region_scope: region::Scope,
+    pub opt_destruction_scope: Option<region::Scope>,
+    pub span: Span,
+    pub stmts: Box<[StmtId]>,
+    pub expr: Option<ExprId>,
+    pub safety_mode: BlockSafety,
+}
+
+#[derive(Copy, Clone, Debug, HashStable)]
+pub enum BlockSafety {
+    Safe,
+    ExplicitUnsafe(hir::HirId),
+    PushUnsafe,
+    PopUnsafe,
+}
+
+#[derive(Debug, HashStable)]
+pub struct Stmt<'tcx> {
+    pub kind: StmtKind<'tcx>,
+    pub opt_destruction_scope: Option<region::Scope>,
+}
+
+#[derive(Debug, HashStable)]
+pub enum StmtKind<'tcx> {
+    Expr {
+        /// scope for this statement; may be used as lifetime of temporaries
+        scope: region::Scope,
+
+        /// expression being evaluated in this statement
+        expr: ExprId,
+    },
+
+    Let {
+        /// scope for variables bound in this let; covers this and
+        /// remaining statements in block
+        remainder_scope: region::Scope,
+
+        /// scope for the initialization itself; might be used as
+        /// lifetime of temporaries
+        init_scope: region::Scope,
+
+        /// `let <PAT> = ...`
+        ///
+        /// if a type is included, it is added as an ascription pattern
+        pattern: Pat<'tcx>,
+
+        /// let pat: ty = <INIT> ...
+        initializer: Option<ExprId>,
+
+        /// the lint level for this let-statement
+        lint_level: LintLevel,
+    },
+}
+
+// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+rustc_data_structures::static_assert_size!(Expr<'_>, 144);
+
+/// The Thir trait implementor lowers their expressions (`&'tcx H::Expr`)
+/// into instances of this `Expr` enum. This lowering can be done
+/// basically as lazily or as eagerly as desired: every recursive
+/// reference to an expression in this enum is an `ExprId`, which
+/// may in turn be another instance of this enum (boxed), or else an
+/// unlowered `&'tcx H::Expr`. Note that instances of `Expr` are very
+/// short-lived. They are created by `Thir::to_expr`, analyzed and
+/// converted into MIR, and then discarded.
+///
+/// If you compare `Expr` to the full compiler AST, you will see it is
+/// a good bit simpler. In fact, a number of the more straight-forward
+/// MIR simplifications are already done in the impl of `Thir`. For
+/// example, method calls and overloaded operators are absent: they are
+/// expected to be converted into `Expr::Call` instances.
+#[derive(Debug, HashStable)]
+pub struct Expr<'tcx> {
+    /// type of this expression
+    pub ty: Ty<'tcx>,
+
+    /// lifetime of this expression if it should be spilled into a
+    /// temporary; should be None only if in a constant context
+    pub temp_lifetime: Option<region::Scope>,
+
+    /// span of the expression in the source
+    pub span: Span,
+
+    /// kind of expression
+    pub kind: ExprKind<'tcx>,
+}
+
+#[derive(Debug, HashStable)]
+pub enum ExprKind<'tcx> {
+    Scope {
+        region_scope: region::Scope,
+        lint_level: LintLevel,
+        value: ExprId,
+    },
+    Box {
+        value: ExprId,
+    },
+    If {
+        cond: ExprId,
+        then: ExprId,
+        else_opt: Option<ExprId>,
+    },
+    Call {
+        ty: Ty<'tcx>,
+        fun: ExprId,
+        args: Box<[ExprId]>,
+        /// Whether this is from a call in HIR, rather than from an overloaded
+        /// operator. `true` for overloaded function call.
+        from_hir_call: bool,
+        /// This `Span` is the span of the function, without the dot and receiver
+        /// (e.g. `foo(a, b)` in `x.foo(a, b)`
+        fn_span: Span,
+    },
+    Deref {
+        arg: ExprId,
+    }, // NOT overloaded!
+    Binary {
+        op: BinOp,
+        lhs: ExprId,
+        rhs: ExprId,
+    }, // NOT overloaded!
+    LogicalOp {
+        op: LogicalOp,
+        lhs: ExprId,
+        rhs: ExprId,
+    }, // NOT overloaded!
+    // LogicalOp is distinct from BinaryOp because of lazy evaluation of the operands.
+    Unary {
+        op: UnOp,
+        arg: ExprId,
+    }, // NOT overloaded!
+    Cast {
+        source: ExprId,
+    },
+    Use {
+        source: ExprId,
+    }, // Use a lexpr to get a vexpr.
+    NeverToAny {
+        source: ExprId,
+    },
+    Pointer {
+        cast: PointerCast,
+        source: ExprId,
+    },
+    Loop {
+        body: ExprId,
+    },
+    Match {
+        scrutinee: ExprId,
+        arms: Box<[ArmId]>,
+    },
+    Block {
+        body: Block,
+    },
+    Assign {
+        lhs: ExprId,
+        rhs: ExprId,
+    },
+    AssignOp {
+        op: BinOp,
+        lhs: ExprId,
+        rhs: ExprId,
+    },
+    Field {
+        lhs: ExprId,
+        name: Field,
+    },
+    Index {
+        lhs: ExprId,
+        index: ExprId,
+    },
+    VarRef {
+        id: hir::HirId,
+    },
+    /// Used to represent upvars mentioned in a closure/generator
+    UpvarRef {
+        /// DefId of the closure/generator
+        closure_def_id: DefId,
+
+        /// HirId of the root variable
+        var_hir_id: hir::HirId,
+    },
+    Borrow {
+        borrow_kind: BorrowKind,
+        arg: ExprId,
+    },
+    /// A `&raw [const|mut] $place_expr` raw borrow resulting in type `*[const|mut] T`.
+    AddressOf {
+        mutability: hir::Mutability,
+        arg: ExprId,
+    },
+    Break {
+        label: region::Scope,
+        value: Option<ExprId>,
+    },
+    Continue {
+        label: region::Scope,
+    },
+    Return {
+        value: Option<ExprId>,
+    },
+    ConstBlock {
+        value: &'tcx Const<'tcx>,
+    },
+    Repeat {
+        value: ExprId,
+        count: &'tcx Const<'tcx>,
+    },
+    Array {
+        fields: Box<[ExprId]>,
+    },
+    Tuple {
+        fields: Box<[ExprId]>,
+    },
+    Adt {
+        adt_def: &'tcx AdtDef,
+        variant_index: VariantIdx,
+        substs: SubstsRef<'tcx>,
+
+        /// Optional user-given substs: for something like `let x =
+        /// Bar::<T> { ... }`.
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+
+        fields: Box<[FieldExpr]>,
+        base: Option<FruInfo<'tcx>>,
+    },
+    PlaceTypeAscription {
+        source: ExprId,
+        /// Type that the user gave to this expression
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+    },
+    ValueTypeAscription {
+        source: ExprId,
+        /// Type that the user gave to this expression
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+    },
+    Closure {
+        closure_id: DefId,
+        substs: UpvarSubsts<'tcx>,
+        upvars: Box<[ExprId]>,
+        movability: Option<hir::Movability>,
+        fake_reads: Vec<(ExprId, FakeReadCause, hir::HirId)>,
+    },
+    Literal {
+        literal: &'tcx Const<'tcx>,
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+        /// The `DefId` of the `const` item this literal
+        /// was produced from, if this is not a user-written
+        /// literal value.
+        const_id: Option<DefId>,
+    },
+    /// A literal containing the address of a `static`.
+    ///
+    /// This is only distinguished from `Literal` so that we can register some
+    /// info for diagnostics.
+    StaticRef {
+        literal: &'tcx Const<'tcx>,
+        def_id: DefId,
+    },
+    InlineAsm {
+        template: &'tcx [InlineAsmTemplatePiece],
+        operands: Box<[InlineAsmOperand<'tcx>]>,
+        options: InlineAsmOptions,
+        line_spans: &'tcx [Span],
+    },
+    /// An expression taking a reference to a thread local.
+    ThreadLocalRef(DefId),
+    LlvmInlineAsm {
+        asm: &'tcx hir::LlvmInlineAsmInner,
+        outputs: Box<[ExprId]>,
+        inputs: Box<[ExprId]>,
+    },
+    Yield {
+        value: ExprId,
+    },
+}
+
+#[derive(Debug, HashStable)]
+pub struct FieldExpr {
+    pub name: Field,
+    pub expr: ExprId,
+}
+
+#[derive(Debug, HashStable)]
+pub struct FruInfo<'tcx> {
+    pub base: ExprId,
+    pub field_types: Box<[Ty<'tcx>]>,
+}
+
+#[derive(Debug, HashStable)]
+pub struct Arm<'tcx> {
+    pub pattern: Pat<'tcx>,
+    pub guard: Option<Guard<'tcx>>,
+    pub body: ExprId,
+    pub lint_level: LintLevel,
+    pub scope: region::Scope,
+    pub span: Span,
+}
+
+#[derive(Debug, HashStable)]
+pub enum Guard<'tcx> {
+    If(ExprId),
+    IfLet(Pat<'tcx>, ExprId),
+}
+
+#[derive(Copy, Clone, Debug, HashStable)]
+pub enum LogicalOp {
+    And,
+    Or,
+}
+
+#[derive(Debug, HashStable)]
+pub enum InlineAsmOperand<'tcx> {
+    In {
+        reg: InlineAsmRegOrRegClass,
+        expr: ExprId,
+    },
+    Out {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        expr: Option<ExprId>,
+    },
+    InOut {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        expr: ExprId,
+    },
+    SplitInOut {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        in_expr: ExprId,
+        out_expr: Option<ExprId>,
+    },
+    Const {
+        value: &'tcx Const<'tcx>,
+        span: Span,
+    },
+    SymFn {
+        expr: ExprId,
+    },
+    SymStatic {
+        def_id: DefId,
+    },
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, HashStable)]
+pub enum BindingMode {
+    ByValue,
+    ByRef(BorrowKind),
+}
+
+#[derive(Clone, Debug, PartialEq, HashStable)]
+pub struct FieldPat<'tcx> {
+    pub field: Field,
+    pub pattern: Pat<'tcx>,
+}
+
+#[derive(Clone, Debug, PartialEq, HashStable)]
+pub struct Pat<'tcx> {
+    pub ty: Ty<'tcx>,
+    pub span: Span,
+    pub kind: Box<PatKind<'tcx>>,
+}
+
+impl<'tcx> Pat<'tcx> {
+    pub fn wildcard_from_ty(ty: Ty<'tcx>) -> Self {
+        Pat { ty, span: DUMMY_SP, kind: Box::new(PatKind::Wild) }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, HashStable)]
+pub struct PatTyProj<'tcx> {
+    pub user_ty: CanonicalUserType<'tcx>,
+}
+
+impl<'tcx> PatTyProj<'tcx> {
+    pub fn from_user_type(user_annotation: CanonicalUserType<'tcx>) -> Self {
+        Self { user_ty: user_annotation }
+    }
+
+    pub fn user_ty(
+        self,
+        annotations: &mut CanonicalUserTypeAnnotations<'tcx>,
+        inferred_ty: Ty<'tcx>,
+        span: Span,
+    ) -> UserTypeProjection {
+        UserTypeProjection {
+            base: annotations.push(CanonicalUserTypeAnnotation {
+                span,
+                user_ty: self.user_ty,
+                inferred_ty,
+            }),
+            projs: Vec::new(),
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, HashStable)]
+pub struct Ascription<'tcx> {
+    pub user_ty: PatTyProj<'tcx>,
+    /// Variance to use when relating the type `user_ty` to the **type of the value being
+    /// matched**. Typically, this is `Variance::Covariant`, since the value being matched must
+    /// have a type that is some subtype of the ascribed type.
+    ///
+    /// Note that this variance does not apply for any bindings within subpatterns. The type
+    /// assigned to those bindings must be exactly equal to the `user_ty` given here.
+    ///
+    /// The only place where this field is not `Covariant` is when matching constants, where
+    /// we currently use `Contravariant` -- this is because the constant type just needs to
+    /// be "comparable" to the type of the input value. So, for example:
+    ///
+    /// ```text
+    /// match x { "foo" => .. }
+    /// ```
+    ///
+    /// requires that `&'static str <: T_x`, where `T_x` is the type of `x`. Really, we should
+    /// probably be checking for a `PartialEq` impl instead, but this preserves the behavior
+    /// of the old type-check for now. See #57280 for details.
+    pub variance: ty::Variance,
+    pub user_ty_span: Span,
+}
+
+#[derive(Clone, Debug, PartialEq, HashStable)]
+pub enum PatKind<'tcx> {
+    Wild,
+
+    AscribeUserType {
+        ascription: Ascription<'tcx>,
+        subpattern: Pat<'tcx>,
+    },
+
+    /// `x`, `ref x`, `x @ P`, etc.
+    Binding {
+        mutability: Mutability,
+        name: Symbol,
+        mode: BindingMode,
+        var: hir::HirId,
+        ty: Ty<'tcx>,
+        subpattern: Option<Pat<'tcx>>,
+        /// Is this the leftmost occurrence of the binding, i.e., is `var` the
+        /// `HirId` of this pattern?
+        is_primary: bool,
+    },
+
+    /// `Foo(...)` or `Foo{...}` or `Foo`, where `Foo` is a variant name from an ADT with
+    /// multiple variants.
+    Variant {
+        adt_def: &'tcx AdtDef,
+        substs: SubstsRef<'tcx>,
+        variant_index: VariantIdx,
+        subpatterns: Vec<FieldPat<'tcx>>,
+    },
+
+    /// `(...)`, `Foo(...)`, `Foo{...}`, or `Foo`, where `Foo` is a variant name from an ADT with
+    /// a single variant.
+    Leaf {
+        subpatterns: Vec<FieldPat<'tcx>>,
+    },
+
+    /// `box P`, `&P`, `&mut P`, etc.
+    Deref {
+        subpattern: Pat<'tcx>,
+    },
+
+    /// One of the following:
+    /// * `&str`, which will be handled as a string pattern and thus exhaustiveness
+    ///   checking will detect if you use the same string twice in different patterns.
+    /// * integer, bool, char or float, which will be handled by exhaustivenes to cover exactly
+    ///   its own value, similar to `&str`, but these values are much simpler.
+    /// * Opaque constants, that must not be matched structurally. So anything that does not derive
+    ///   `PartialEq` and `Eq`.
+    Constant {
+        value: &'tcx ty::Const<'tcx>,
+    },
+
+    Range(PatRange<'tcx>),
+
+    /// Matches against a slice, checking the length and extracting elements.
+    /// irrefutable when there is a slice pattern and both `prefix` and `suffix` are empty.
+    /// e.g., `&[ref xs @ ..]`.
+    Slice {
+        prefix: Vec<Pat<'tcx>>,
+        slice: Option<Pat<'tcx>>,
+        suffix: Vec<Pat<'tcx>>,
+    },
+
+    /// Fixed match against an array; irrefutable.
+    Array {
+        prefix: Vec<Pat<'tcx>>,
+        slice: Option<Pat<'tcx>>,
+        suffix: Vec<Pat<'tcx>>,
+    },
+
+    /// An or-pattern, e.g. `p | q`.
+    /// Invariant: `pats.len() >= 2`.
+    Or {
+        pats: Vec<Pat<'tcx>>,
+    },
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, HashStable)]
+pub struct PatRange<'tcx> {
+    pub lo: &'tcx ty::Const<'tcx>,
+    pub hi: &'tcx ty::Const<'tcx>,
+    pub end: RangeEnd,
+}
+
+impl<'tcx> fmt::Display for Pat<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Printing lists is a chore.
+        let mut first = true;
+        let mut start_or_continue = |s| {
+            if first {
+                first = false;
+                ""
+            } else {
+                s
+            }
+        };
+        let mut start_or_comma = || start_or_continue(", ");
+
+        match *self.kind {
+            PatKind::Wild => write!(f, "_"),
+            PatKind::AscribeUserType { ref subpattern, .. } => write!(f, "{}: _", subpattern),
+            PatKind::Binding { mutability, name, mode, ref subpattern, .. } => {
+                let is_mut = match mode {
+                    BindingMode::ByValue => mutability == Mutability::Mut,
+                    BindingMode::ByRef(bk) => {
+                        write!(f, "ref ")?;
+                        matches!(bk, BorrowKind::Mut { .. })
+                    }
+                };
+                if is_mut {
+                    write!(f, "mut ")?;
+                }
+                write!(f, "{}", name)?;
+                if let Some(ref subpattern) = *subpattern {
+                    write!(f, " @ {}", subpattern)?;
+                }
+                Ok(())
+            }
+            PatKind::Variant { ref subpatterns, .. } | PatKind::Leaf { ref subpatterns } => {
+                let variant = match *self.kind {
+                    PatKind::Variant { adt_def, variant_index, .. } => {
+                        Some(&adt_def.variants[variant_index])
+                    }
+                    _ => {
+                        if let ty::Adt(adt, _) = self.ty.kind() {
+                            if !adt.is_enum() {
+                                Some(&adt.variants[VariantIdx::new(0)])
+                            } else {
+                                None
+                            }
+                        } else {
+                            None
+                        }
+                    }
+                };
+
+                if let Some(variant) = variant {
+                    write!(f, "{}", variant.ident)?;
+
+                    // Only for Adt we can have `S {...}`,
+                    // which we handle separately here.
+                    if variant.ctor_kind == CtorKind::Fictive {
+                        write!(f, " {{ ")?;
+
+                        let mut printed = 0;
+                        for p in subpatterns {
+                            if let PatKind::Wild = *p.pattern.kind {
+                                continue;
+                            }
+                            let name = variant.fields[p.field.index()].ident;
+                            write!(f, "{}{}: {}", start_or_comma(), name, p.pattern)?;
+                            printed += 1;
+                        }
+
+                        if printed < variant.fields.len() {
+                            write!(f, "{}..", start_or_comma())?;
+                        }
+
+                        return write!(f, " }}");
+                    }
+                }
+
+                let num_fields = variant.map_or(subpatterns.len(), |v| v.fields.len());
+                if num_fields != 0 || variant.is_none() {
+                    write!(f, "(")?;
+                    for i in 0..num_fields {
+                        write!(f, "{}", start_or_comma())?;
+
+                        // Common case: the field is where we expect it.
+                        if let Some(p) = subpatterns.get(i) {
+                            if p.field.index() == i {
+                                write!(f, "{}", p.pattern)?;
+                                continue;
+                            }
+                        }
+
+                        // Otherwise, we have to go looking for it.
+                        if let Some(p) = subpatterns.iter().find(|p| p.field.index() == i) {
+                            write!(f, "{}", p.pattern)?;
+                        } else {
+                            write!(f, "_")?;
+                        }
+                    }
+                    write!(f, ")")?;
+                }
+
+                Ok(())
+            }
+            PatKind::Deref { ref subpattern } => {
+                match self.ty.kind() {
+                    ty::Adt(def, _) if def.is_box() => write!(f, "box ")?,
+                    ty::Ref(_, _, mutbl) => {
+                        write!(f, "&{}", mutbl.prefix_str())?;
+                    }
+                    _ => bug!("{} is a bad Deref pattern type", self.ty),
+                }
+                write!(f, "{}", subpattern)
+            }
+            PatKind::Constant { value } => write!(f, "{}", value),
+            PatKind::Range(PatRange { lo, hi, end }) => {
+                write!(f, "{}", lo)?;
+                write!(f, "{}", end)?;
+                write!(f, "{}", hi)
+            }
+            PatKind::Slice { ref prefix, ref slice, ref suffix }
+            | PatKind::Array { ref prefix, ref slice, ref suffix } => {
+                write!(f, "[")?;
+                for p in prefix {
+                    write!(f, "{}{}", start_or_comma(), p)?;
+                }
+                if let Some(ref slice) = *slice {
+                    write!(f, "{}", start_or_comma())?;
+                    match *slice.kind {
+                        PatKind::Wild => {}
+                        _ => write!(f, "{}", slice)?,
+                    }
+                    write!(f, "..")?;
+                }
+                for p in suffix {
+                    write!(f, "{}{}", start_or_comma(), p)?;
+                }
+                write!(f, "]")
+            }
+            PatKind::Or { ref pats } => {
+                for pat in pats {
+                    write!(f, "{}{}", start_or_continue(" | "), pat)?;
+                }
+                Ok(())
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 790463d6fc6..cb08d7671bd 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -13,6 +13,7 @@ use crate::middle::resolve_lifetime::{self, LifetimeScopeForPath, ObjectLifetime
 use crate::middle::stability;
 use crate::mir::interpret::{self, Allocation, ConstValue, Scalar};
 use crate::mir::{Body, Field, Local, Place, PlaceElem, ProjectionKind, Promoted};
+use crate::thir::Thir;
 use crate::traits;
 use crate::ty::query::{self, OnDiskCache, TyCtxtAt};
 use crate::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, Subst, SubstsRef, UserSubsts};
@@ -1041,6 +1042,10 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    pub fn alloc_steal_thir(self, thir: Thir<'tcx>) -> &'tcx Steal<Thir<'tcx>> {
+        self.arena.alloc(Steal::new(thir))
+    }
+
     pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> &'tcx Steal<Body<'tcx>> {
         self.arena.alloc(Steal::new(mir))
     }
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index 3c772a14647..3bdb438896b 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -18,6 +18,7 @@ use crate::mir::interpret::GlobalId;
 use crate::mir::interpret::{ConstAlloc, LitToConstError, LitToConstInput};
 use crate::mir::interpret::{ConstValue, EvalToAllocationRawResult, EvalToConstValueResult};
 use crate::mir::mono::CodegenUnit;
+use crate::thir;
 use crate::traits::query::{
     CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
     CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal,
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 691bfcc98d1..f35ecb4d3cd 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -669,7 +669,7 @@ impl<'tcx> GeneratorSubsts<'tcx> {
     }
 }
 
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, HashStable)]
 pub enum UpvarSubsts<'tcx> {
     Closure(SubstsRef<'tcx>),
     Generator(SubstsRef<'tcx>),
diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
index ea3a5174fd8..8426b24270d 100644
--- a/compiler/rustc_mir_build/src/build/block.rs
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -1,8 +1,8 @@
 use crate::build::matches::ArmHasGuard;
 use crate::build::ForGuard::OutsideGuard;
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
-use crate::thir::*;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 use rustc_session::lint::builtin::UNSAFE_OP_IN_UNSAFE_FN;
 use rustc_session::lint::Level;
 use rustc_span::Span;
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index 796a90713ba..5e305ebba2f 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -1,8 +1,8 @@
 //! See docs in build/expr/mod.rs
 
 use crate::build::Builder;
-use crate::thir::*;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 use rustc_middle::ty::CanonicalUserTypeAnnotation;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
index 1c439aad394..b2a1dbf4c52 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_operand.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
@@ -2,9 +2,9 @@
 
 use crate::build::expr::category::Category;
 use crate::build::{BlockAnd, BlockAndExtension, Builder};
-use crate::thir::*;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Returns an operand suitable for use until the end of the current
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index 96df77a65da..842d7666742 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -3,13 +3,13 @@
 use crate::build::expr::category::Category;
 use crate::build::ForGuard::{OutsideGuard, RefWithinGuard};
 use crate::build::{BlockAnd, BlockAndExtension, Builder};
-use crate::thir::*;
 use rustc_hir::def_id::DefId;
 use rustc_hir::HirId;
 use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
 use rustc_middle::middle::region;
 use rustc_middle::mir::AssertKind::BoundsCheck;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 use rustc_middle::ty::AdtDef;
 use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, Variance};
 use rustc_span::Span;
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index 92a2a7bc17a..2eb6597e81d 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -5,11 +5,11 @@ use rustc_index::vec::Idx;
 use crate::build::expr::as_place::PlaceBase;
 use crate::build::expr::category::{Category, RvalueFunc};
 use crate::build::{BlockAnd, BlockAndExtension, Builder};
-use crate::thir::*;
 use rustc_middle::middle::region;
 use rustc_middle::mir::AssertKind;
 use rustc_middle::mir::Place;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 use rustc_middle::ty::{self, Ty, UpvarSubsts};
 use rustc_span::Span;
 
diff --git a/compiler/rustc_mir_build/src/build/expr/as_temp.rs b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
index 96bf3e6d69d..45e0243c88a 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_temp.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
@@ -2,10 +2,10 @@
 
 use crate::build::scope::DropKind;
 use crate::build::{BlockAnd, BlockAndExtension, Builder};
-use crate::thir::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr` into a fresh temporary. This is used when building
diff --git a/compiler/rustc_mir_build/src/build/expr/category.rs b/compiler/rustc_mir_build/src/build/expr/category.rs
index 9320b5810e3..c834ce6ce68 100644
--- a/compiler/rustc_mir_build/src/build/expr/category.rs
+++ b/compiler/rustc_mir_build/src/build/expr/category.rs
@@ -1,4 +1,4 @@
-use crate::thir::*;
+use rustc_middle::thir::*;
 
 #[derive(Debug, PartialEq)]
 crate enum Category {
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index d7c8a07103e..f2b00f0f6ed 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -2,13 +2,13 @@
 
 use crate::build::expr::category::{Category, RvalueFunc};
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
-use crate::thir::*;
 use rustc_ast::InlineAsmOptions;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
 use rustc_index::vec::Idx;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 use rustc_middle::ty::{self, CanonicalUserTypeAnnotation};
 use std::iter;
 
@@ -337,8 +337,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 block.unit()
             }
             ExprKind::InlineAsm { template, ref operands, options, line_spans } => {
-                use crate::thir;
-                use rustc_middle::mir;
+                use rustc_middle::{mir, thir};
                 let operands = operands
                     .into_iter()
                     .map(|op| match *op {
diff --git a/compiler/rustc_mir_build/src/build/expr/stmt.rs b/compiler/rustc_mir_build/src/build/expr/stmt.rs
index d2442f33b0c..b03a6bb1a2b 100644
--- a/compiler/rustc_mir_build/src/build/expr/stmt.rs
+++ b/compiler/rustc_mir_build/src/build/expr/stmt.rs
@@ -1,8 +1,8 @@
 use crate::build::scope::BreakableTarget;
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
-use crate::thir::*;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Builds a block of MIR statements to evaluate the THIR `expr`.
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index c30193b5a5a..8164529dd1f 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -10,7 +10,6 @@ use crate::build::scope::DropKind;
 use crate::build::ForGuard::{self, OutsideGuard, RefWithinGuard};
 use crate::build::{BlockAnd, BlockAndExtension, Builder};
 use crate::build::{GuardFrame, GuardFrameLocal, LocalsForNode};
-use crate::thir::{self, *};
 use rustc_data_structures::{
     fx::{FxHashSet, FxIndexMap},
     stack::ensure_sufficient_stack,
@@ -19,6 +18,7 @@ use rustc_hir::HirId;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
+use rustc_middle::thir::{self, *};
 use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
@@ -432,7 +432,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         ..
                     },
                 ascription:
-                    thir::pattern::Ascription { user_ty: pat_ascription_ty, variance: _, user_ty_span },
+                    thir::Ascription { user_ty: pat_ascription_ty, variance: _, user_ty_span },
             } => {
                 let place =
                     self.storage_live_binding(block, var, irrefutable_pat.span, OutsideGuard, true);
@@ -687,7 +687,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             PatKind::AscribeUserType {
                 ref subpattern,
-                ascription: thir::pattern::Ascription { ref user_ty, user_ty_span, variance: _ },
+                ascription: thir::Ascription { ref user_ty, user_ty_span, variance: _ },
             } => {
                 // This corresponds to something like
                 //
diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs
index 3ad143a57ff..13cfc3695cc 100644
--- a/compiler/rustc_mir_build/src/build/matches/simplify.rs
+++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs
@@ -15,8 +15,8 @@
 use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::{Ascription, Binding, Candidate, MatchPair};
 use crate::build::Builder;
-use crate::thir::{self, *};
 use rustc_hir::RangeEnd;
+use rustc_middle::thir::{self, *};
 use rustc_middle::ty;
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_target::abi::{Integer, Size};
@@ -152,7 +152,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match *match_pair.pattern.kind {
             PatKind::AscribeUserType {
                 ref subpattern,
-                ascription: thir::pattern::Ascription { variance, user_ty, user_ty_span },
+                ascription: thir::Ascription { variance, user_ty, user_ty_span },
             } => {
                 // Apply the type ascription to the value at `match_pair.place`, which is the
                 candidate.ascriptions.push(Ascription {
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index b082169cd63..c87f42738c6 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -9,11 +9,11 @@ use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::{Candidate, MatchPair, Test, TestKind};
 use crate::build::Builder;
 use crate::thir::pattern::compare_const_vals;
-use crate::thir::*;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::{LangItem, RangeEnd};
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 use rustc_middle::ty::subst::{GenericArg, Subst};
 use rustc_middle::ty::util::IntTypeExt;
 use rustc_middle::ty::{self, adjustment::PointerCast, Ty, TyCtxt};
diff --git a/compiler/rustc_mir_build/src/build/matches/util.rs b/compiler/rustc_mir_build/src/build/matches/util.rs
index d49a00a5660..3cf8ae6efd9 100644
--- a/compiler/rustc_mir_build/src/build/matches/util.rs
+++ b/compiler/rustc_mir_build/src/build/matches/util.rs
@@ -1,8 +1,8 @@
 use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::MatchPair;
 use crate::build::Builder;
-use crate::thir::*;
 use rustc_middle::mir::*;
+use rustc_middle::thir::*;
 use rustc_middle::ty;
 use smallvec::SmallVec;
 use std::convert::TryInto;
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index 4f6c57be2da..d8f1f5b97ee 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -1,7 +1,7 @@
 use crate::build;
 use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::scope::DropKind;
-use crate::thir::{build_thir, BindingMode, Expr, ExprId, LintLevel, Pat, PatKind, Thir};
+use crate::thir::pattern::pat_from_hir;
 use rustc_attr::{self as attr, UnwindAttr};
 use rustc_errors::ErrorReported;
 use rustc_hir as hir;
@@ -13,6 +13,7 @@ use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
+use rustc_middle::thir::{BindingMode, Expr, ExprId, LintLevel, PatKind, Thir};
 use rustc_middle::ty::subst::Subst;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeckResults};
 use rustc_span::symbol::{kw, sym};
@@ -45,6 +46,16 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
     let body_owner_kind = tcx.hir().body_owner_kind(id);
     let typeck_results = tcx.typeck_opt_const_arg(def);
 
+    // Ensure unsafeck is ran before we steal the THIR.
+    match def {
+        ty::WithOptConstParam { did, const_param_did: Some(const_param_did) } => {
+            tcx.ensure().thir_check_unsafety_for_const_arg((did, const_param_did))
+        }
+        ty::WithOptConstParam { did, const_param_did: None } => {
+            tcx.ensure().thir_check_unsafety(did)
+        }
+    }
+
     // Figure out what primary body this item has.
     let (body_id, return_ty_span, span_with_body) = match tcx.hir().get(id) {
         Node::Expr(hir::Expr { kind: hir::ExprKind::Closure(_, decl, body_id, _, _), .. }) => {
@@ -104,7 +115,10 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
             };
 
             let body = tcx.hir().body(body_id);
-            let (thir, expr) = build_thir(tcx, def, &body.value);
+            let (thir, expr) = tcx.thir_body(def);
+            // We ran all queries that depended on THIR at the beginning
+            // of `mir_build`, so now we can steal it
+            let thir = thir.steal();
             let ty = tcx.type_of(fn_def_id);
             let mut abi = fn_sig.abi;
             let implicit_argument = match ty.kind() {
@@ -212,8 +226,10 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
 
             let return_ty = typeck_results.node_type(id);
 
-            let ast_expr = &tcx.hir().body(body_id).value;
-            let (thir, expr) = build_thir(tcx, def, ast_expr);
+            let (thir, expr) = tcx.thir_body(def);
+            // We ran all queries that depended on THIR at the beginning
+            // of `mir_build`, so now we can steal it
+            let thir = thir.steal();
 
             build::construct_const(&thir, &infcx, expr, def, id, return_ty, return_ty_span)
         };
@@ -1016,7 +1032,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     Node::Pat(pat) | Node::Binding(pat) => pat,
                     node => bug!("pattern became {:?}", node),
                 };
-                let pattern = Pat::from_hir(tcx, self.param_env, self.typeck_results, pat);
+                let pattern = pat_from_hir(tcx, self.param_env, self.typeck_results, pat);
                 let original_source_scope = self.source_scope;
                 let span = pattern.span;
                 self.set_correct_source_scope_for_arg(arg.hir_id, original_source_scope, span);
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index e79a19d57ac..3de894bd370 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -82,11 +82,12 @@ that contains only loops and breakable blocks. It tracks where a `break`,
 */
 
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
-use crate::thir::{Expr, LintLevel};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::vec::IndexVec;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
+use rustc_middle::thir::{Expr, LintLevel};
+
 use rustc_span::{Span, DUMMY_SP};
 
 #[derive(Debug)]
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
index 0aa4d0fe2cb..7e64c5f189e 100644
--- a/compiler/rustc_mir_build/src/check_unsafety.rs
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -1,8 +1,8 @@
 use crate::thir::visit::{self, Visitor};
-use crate::thir::*;
 
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
+use rustc_middle::thir::*;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE};
 use rustc_session::lint::Level;
@@ -328,13 +328,20 @@ impl UnsafeOpKind {
 
 // FIXME: checking unsafety for closures should be handled by their parent body,
 // as they inherit their "safety context" from their declaration site.
-pub fn check_unsafety<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    thir: &Thir<'tcx>,
-    expr: ExprId,
-    def_id: LocalDefId,
-    hir_id: hir::HirId,
-) {
+pub fn check_unsafety<'tcx>(tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>) {
+    // THIR unsafeck is gated under `-Z thir-unsafeck`
+    if !tcx.sess.opts.debugging_opts.thir_unsafeck {
+        return;
+    }
+
+    let (thir, expr) = tcx.thir_body(def);
+    let thir = &thir.borrow();
+    // If `thir` is empty, a type error occured, skip this body.
+    if thir.exprs.is_empty() {
+        return;
+    }
+
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
     let body_unsafety = tcx.hir().fn_sig_by_hir_id(hir_id).map_or(BodyUnsafety::Safe, |fn_sig| {
         if fn_sig.header.unsafety == hir::Unsafety::Unsafe {
             BodyUnsafety::Unsafe(fn_sig.span)
@@ -342,12 +349,12 @@ pub fn check_unsafety<'tcx>(
             BodyUnsafety::Safe
         }
     });
-    let body_target_features = &tcx.codegen_fn_attrs(def_id).target_features;
+    let body_target_features = &tcx.codegen_fn_attrs(def.did).target_features;
     let safety_context =
         if body_unsafety.is_unsafe() { SafetyContext::UnsafeFn } else { SafetyContext::Safe };
     let is_const = match tcx.hir().body_owner_kind(hir_id) {
         hir::BodyOwnerKind::Closure => false,
-        hir::BodyOwnerKind::Fn => tcx.is_const_fn_raw(def_id.to_def_id()),
+        hir::BodyOwnerKind::Fn => tcx.is_const_fn_raw(def.did.to_def_id()),
         hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_) => true,
     };
     let mut visitor = UnsafetyVisitor {
@@ -362,22 +369,11 @@ pub fn check_unsafety<'tcx>(
     visitor.visit_expr(&thir[expr]);
 }
 
-crate fn thir_check_unsafety_inner<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    def: ty::WithOptConstParam<LocalDefId>,
-) {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
-    let body_id = tcx.hir().body_owned_by(hir_id);
-    let body = tcx.hir().body(body_id);
-    let (thir, expr) = cx::build_thir(tcx, def, &body.value);
-    check_unsafety(tcx, &thir, expr, def.did, hir_id);
-}
-
 crate fn thir_check_unsafety<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) {
     if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) {
         tcx.thir_check_unsafety_for_const_arg(def)
     } else {
-        thir_check_unsafety_inner(tcx, ty::WithOptConstParam::unknown(def_id))
+        check_unsafety(tcx, ty::WithOptConstParam::unknown(def_id))
     }
 }
 
@@ -385,5 +381,5 @@ crate fn thir_check_unsafety_for_const_arg<'tcx>(
     tcx: TyCtxt<'tcx>,
     (did, param_did): (LocalDefId, DefId),
 ) {
-    thir_check_unsafety_inner(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) })
+    check_unsafety(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) })
 }
diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs
index d4e9a0a3169..67455beb07c 100644
--- a/compiler/rustc_mir_build/src/lib.rs
+++ b/compiler/rustc_mir_build/src/lib.rs
@@ -31,4 +31,5 @@ pub fn provide(providers: &mut Providers) {
     providers.mir_built = build::mir_built;
     providers.thir_check_unsafety = check_unsafety::thir_check_unsafety;
     providers.thir_check_unsafety_for_const_arg = check_unsafety::thir_check_unsafety_for_const_arg;
+    providers.thir_body = thir::cx::thir_body;
 }
diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
index b90f9abe33a..77235fe9ab3 100644
--- a/compiler/rustc_mir_build/src/thir/cx/block.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -1,8 +1,8 @@
 use crate::thir::cx::Cx;
-use crate::thir::{self, *};
 
 use rustc_hir as hir;
 use rustc_middle::middle::region;
+use rustc_middle::thir::*;
 use rustc_middle::ty;
 
 use rustc_index::vec::Idx;
@@ -81,7 +81,7 @@ impl<'tcx> Cx<'tcx> {
                                     ty: pattern.ty,
                                     span: pattern.span,
                                     kind: Box::new(PatKind::AscribeUserType {
-                                        ascription: thir::pattern::Ascription {
+                                        ascription: Ascription {
                                             user_ty: PatTyProj::from_user_type(user_ty),
                                             user_ty_span: ty.span,
                                             variance: ty::Variance::Covariant,
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 9cc7fbdf824..aa4acfab5c8 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -1,6 +1,5 @@
 use crate::thir::cx::Cx;
 use crate::thir::util::UserAnnotatedTyHelpers;
-use crate::thir::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
@@ -8,14 +7,18 @@ use rustc_index::vec::Idx;
 use rustc_middle::hir::place::Place as HirPlace;
 use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
 use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
+use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::Scalar;
-use rustc_middle::mir::BorrowKind;
+use rustc_middle::mir::{BinOp, BorrowKind, Field, UnOp};
+use rustc_middle::thir::*;
 use rustc_middle::ty::adjustment::{
     Adjust, Adjustment, AutoBorrow, AutoBorrowMutability, PointerCast,
 };
 use rustc_middle::ty::subst::{InternalSubsts, SubstsRef};
-use rustc_middle::ty::{self, AdtKind, Ty};
+use rustc_middle::ty::{self, AdtKind, Ty, UpvarSubsts, UserType};
+use rustc_span::def_id::DefId;
 use rustc_span::Span;
+use rustc_target::abi::VariantIdx;
 
 impl<'tcx> Cx<'tcx> {
     crate fn mirror_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) -> ExprId {
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
index aad6319e404..49ba71e3520 100644
--- a/compiler/rustc_mir_build/src/thir/cx/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -2,25 +2,32 @@
 //! structures into the THIR. The `builder` is generally ignorant of the tcx,
 //! etc., and instead goes through the `Cx` for most of its work.
 
+use crate::thir::pattern::pat_from_hir;
 use crate::thir::util::UserAnnotatedTyHelpers;
-use crate::thir::*;
 
 use rustc_ast as ast;
+use rustc_data_structures::steal::Steal;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::Node;
 use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::{LitToConstError, LitToConstInput};
+use rustc_middle::thir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::Span;
 
-pub fn build_thir<'tcx>(
+crate fn thir_body<'tcx>(
     tcx: TyCtxt<'tcx>,
     owner_def: ty::WithOptConstParam<LocalDefId>,
-    expr: &'tcx hir::Expr<'tcx>,
-) -> (Thir<'tcx>, ExprId) {
+) -> (&'tcx Steal<Thir<'tcx>>, ExprId) {
+    let hir = tcx.hir();
+    let body = hir.body(hir.body_owned_by(hir.local_def_id_to_hir_id(owner_def.did)));
     let mut cx = Cx::new(tcx, owner_def);
-    let expr = cx.mirror_expr(expr);
-    (cx.thir, expr)
+    if cx.typeck_results.tainted_by_errors.is_some() {
+        return (tcx.alloc_steal_thir(Thir::new()), ExprId::from_u32(0));
+    }
+    let expr = cx.mirror_expr(&body.value);
+    (tcx.alloc_steal_thir(cx.thir), expr)
 }
 
 struct Cx<'tcx> {
@@ -79,7 +86,7 @@ impl<'tcx> Cx<'tcx> {
             Node::Pat(p) | Node::Binding(p) => p,
             node => bug!("pattern became {:?}", node),
         };
-        Pat::from_hir(self.tcx, self.param_env, self.typeck_results(), p)
+        pat_from_hir(self.tcx, self.param_env, self.typeck_results(), p)
     }
 }
 
diff --git a/compiler/rustc_mir_build/src/thir/mod.rs b/compiler/rustc_mir_build/src/thir/mod.rs
index d188d17dd56..e5123d8ef0c 100644
--- a/compiler/rustc_mir_build/src/thir/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/mod.rs
@@ -4,438 +4,11 @@
 //! unit-tested and separated from the Rust source and compiler data
 //! structures.
 
-use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
-use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
-use rustc_index::newtype_index;
-use rustc_index::vec::IndexVec;
-use rustc_middle::infer::canonical::Canonical;
-use rustc_middle::middle::region;
-use rustc_middle::mir::{BinOp, BorrowKind, FakeReadCause, Field, UnOp};
-use rustc_middle::ty::adjustment::PointerCast;
-use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{AdtDef, Const, Ty, UpvarSubsts, UserType};
-use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
-use rustc_target::asm::InlineAsmRegOrRegClass;
-
-use std::ops::Index;
-
 crate mod constant;
 
 crate mod cx;
-pub use cx::build_thir;
 
 crate mod pattern;
-pub use self::pattern::{Ascription, BindingMode, FieldPat, Pat, PatKind, PatRange, PatTyProj};
 
 mod util;
 pub mod visit;
-
-newtype_index! {
-    pub struct ArmId {
-        DEBUG_FORMAT = "a{}"
-    }
-}
-
-newtype_index! {
-    pub struct ExprId {
-        DEBUG_FORMAT = "e{}"
-    }
-}
-
-newtype_index! {
-    pub struct StmtId {
-        DEBUG_FORMAT = "s{}"
-    }
-}
-
-macro_rules! thir_with_elements {
-    ($($name:ident: $id:ty => $value:ty,)*) => {
-        pub struct Thir<'tcx> {
-            $(
-                $name: IndexVec<$id, $value>,
-            )*
-        }
-
-        impl<'tcx> Thir<'tcx> {
-            fn new() -> Thir<'tcx> {
-                Thir {
-                    $(
-                        $name: IndexVec::new(),
-                    )*
-                }
-            }
-        }
-
-        $(
-            impl<'tcx> Index<$id> for Thir<'tcx> {
-                type Output = $value;
-                fn index(&self, index: $id) -> &Self::Output {
-                    &self.$name[index]
-                }
-            }
-        )*
-    }
-}
-
-thir_with_elements! {
-    arms: ArmId => Arm<'tcx>,
-    exprs: ExprId => Expr<'tcx>,
-    stmts: StmtId => Stmt<'tcx>,
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum LintLevel {
-    Inherited,
-    Explicit(hir::HirId),
-}
-
-#[derive(Debug)]
-pub struct Block {
-    pub targeted_by_break: bool,
-    pub region_scope: region::Scope,
-    pub opt_destruction_scope: Option<region::Scope>,
-    pub span: Span,
-    pub stmts: Box<[StmtId]>,
-    pub expr: Option<ExprId>,
-    pub safety_mode: BlockSafety,
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum BlockSafety {
-    Safe,
-    ExplicitUnsafe(hir::HirId),
-    PushUnsafe,
-    PopUnsafe,
-}
-
-#[derive(Debug)]
-pub struct Stmt<'tcx> {
-    pub kind: StmtKind<'tcx>,
-    pub opt_destruction_scope: Option<region::Scope>,
-}
-
-#[derive(Debug)]
-pub enum StmtKind<'tcx> {
-    Expr {
-        /// scope for this statement; may be used as lifetime of temporaries
-        scope: region::Scope,
-
-        /// expression being evaluated in this statement
-        expr: ExprId,
-    },
-
-    Let {
-        /// scope for variables bound in this let; covers this and
-        /// remaining statements in block
-        remainder_scope: region::Scope,
-
-        /// scope for the initialization itself; might be used as
-        /// lifetime of temporaries
-        init_scope: region::Scope,
-
-        /// `let <PAT> = ...`
-        ///
-        /// if a type is included, it is added as an ascription pattern
-        pattern: Pat<'tcx>,
-
-        /// let pat: ty = <INIT> ...
-        initializer: Option<ExprId>,
-
-        /// the lint level for this let-statement
-        lint_level: LintLevel,
-    },
-}
-
-// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Expr<'_>, 144);
-
-/// The Thir trait implementor lowers their expressions (`&'tcx H::Expr`)
-/// into instances of this `Expr` enum. This lowering can be done
-/// basically as lazily or as eagerly as desired: every recursive
-/// reference to an expression in this enum is an `ExprId`, which
-/// may in turn be another instance of this enum (boxed), or else an
-/// unlowered `&'tcx H::Expr`. Note that instances of `Expr` are very
-/// short-lived. They are created by `Thir::to_expr`, analyzed and
-/// converted into MIR, and then discarded.
-///
-/// If you compare `Expr` to the full compiler AST, you will see it is
-/// a good bit simpler. In fact, a number of the more straight-forward
-/// MIR simplifications are already done in the impl of `Thir`. For
-/// example, method calls and overloaded operators are absent: they are
-/// expected to be converted into `Expr::Call` instances.
-#[derive(Debug)]
-pub struct Expr<'tcx> {
-    /// type of this expression
-    pub ty: Ty<'tcx>,
-
-    /// lifetime of this expression if it should be spilled into a
-    /// temporary; should be None only if in a constant context
-    pub temp_lifetime: Option<region::Scope>,
-
-    /// span of the expression in the source
-    pub span: Span,
-
-    /// kind of expression
-    pub kind: ExprKind<'tcx>,
-}
-
-#[derive(Debug)]
-pub enum ExprKind<'tcx> {
-    Scope {
-        region_scope: region::Scope,
-        lint_level: LintLevel,
-        value: ExprId,
-    },
-    Box {
-        value: ExprId,
-    },
-    If {
-        cond: ExprId,
-        then: ExprId,
-        else_opt: Option<ExprId>,
-    },
-    Call {
-        ty: Ty<'tcx>,
-        fun: ExprId,
-        args: Box<[ExprId]>,
-        /// Whether this is from a call in HIR, rather than from an overloaded
-        /// operator. `true` for overloaded function call.
-        from_hir_call: bool,
-        /// This `Span` is the span of the function, without the dot and receiver
-        /// (e.g. `foo(a, b)` in `x.foo(a, b)`
-        fn_span: Span,
-    },
-    Deref {
-        arg: ExprId,
-    }, // NOT overloaded!
-    Binary {
-        op: BinOp,
-        lhs: ExprId,
-        rhs: ExprId,
-    }, // NOT overloaded!
-    LogicalOp {
-        op: LogicalOp,
-        lhs: ExprId,
-        rhs: ExprId,
-    }, // NOT overloaded!
-    // LogicalOp is distinct from BinaryOp because of lazy evaluation of the operands.
-    Unary {
-        op: UnOp,
-        arg: ExprId,
-    }, // NOT overloaded!
-    Cast {
-        source: ExprId,
-    },
-    Use {
-        source: ExprId,
-    }, // Use a lexpr to get a vexpr.
-    NeverToAny {
-        source: ExprId,
-    },
-    Pointer {
-        cast: PointerCast,
-        source: ExprId,
-    },
-    Loop {
-        body: ExprId,
-    },
-    Match {
-        scrutinee: ExprId,
-        arms: Box<[ArmId]>,
-    },
-    Block {
-        body: Block,
-    },
-    Assign {
-        lhs: ExprId,
-        rhs: ExprId,
-    },
-    AssignOp {
-        op: BinOp,
-        lhs: ExprId,
-        rhs: ExprId,
-    },
-    Field {
-        lhs: ExprId,
-        name: Field,
-    },
-    Index {
-        lhs: ExprId,
-        index: ExprId,
-    },
-    VarRef {
-        id: hir::HirId,
-    },
-    /// Used to represent upvars mentioned in a closure/generator
-    UpvarRef {
-        /// DefId of the closure/generator
-        closure_def_id: DefId,
-
-        /// HirId of the root variable
-        var_hir_id: hir::HirId,
-    },
-    Borrow {
-        borrow_kind: BorrowKind,
-        arg: ExprId,
-    },
-    /// A `&raw [const|mut] $place_expr` raw borrow resulting in type `*[const|mut] T`.
-    AddressOf {
-        mutability: hir::Mutability,
-        arg: ExprId,
-    },
-    Break {
-        label: region::Scope,
-        value: Option<ExprId>,
-    },
-    Continue {
-        label: region::Scope,
-    },
-    Return {
-        value: Option<ExprId>,
-    },
-    ConstBlock {
-        value: &'tcx Const<'tcx>,
-    },
-    Repeat {
-        value: ExprId,
-        count: &'tcx Const<'tcx>,
-    },
-    Array {
-        fields: Box<[ExprId]>,
-    },
-    Tuple {
-        fields: Box<[ExprId]>,
-    },
-    Adt {
-        adt_def: &'tcx AdtDef,
-        variant_index: VariantIdx,
-        substs: SubstsRef<'tcx>,
-
-        /// Optional user-given substs: for something like `let x =
-        /// Bar::<T> { ... }`.
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
-
-        fields: Box<[FieldExpr]>,
-        base: Option<FruInfo<'tcx>>,
-    },
-    PlaceTypeAscription {
-        source: ExprId,
-        /// Type that the user gave to this expression
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
-    },
-    ValueTypeAscription {
-        source: ExprId,
-        /// Type that the user gave to this expression
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
-    },
-    Closure {
-        closure_id: DefId,
-        substs: UpvarSubsts<'tcx>,
-        upvars: Box<[ExprId]>,
-        movability: Option<hir::Movability>,
-        fake_reads: Vec<(ExprId, FakeReadCause, hir::HirId)>,
-    },
-    Literal {
-        literal: &'tcx Const<'tcx>,
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
-        /// The `DefId` of the `const` item this literal
-        /// was produced from, if this is not a user-written
-        /// literal value.
-        const_id: Option<DefId>,
-    },
-    /// A literal containing the address of a `static`.
-    ///
-    /// This is only distinguished from `Literal` so that we can register some
-    /// info for diagnostics.
-    StaticRef {
-        literal: &'tcx Const<'tcx>,
-        def_id: DefId,
-    },
-    InlineAsm {
-        template: &'tcx [InlineAsmTemplatePiece],
-        operands: Box<[InlineAsmOperand<'tcx>]>,
-        options: InlineAsmOptions,
-        line_spans: &'tcx [Span],
-    },
-    /// An expression taking a reference to a thread local.
-    ThreadLocalRef(DefId),
-    LlvmInlineAsm {
-        asm: &'tcx hir::LlvmInlineAsmInner,
-        outputs: Box<[ExprId]>,
-        inputs: Box<[ExprId]>,
-    },
-    Yield {
-        value: ExprId,
-    },
-}
-
-#[derive(Debug)]
-pub struct FieldExpr {
-    pub name: Field,
-    pub expr: ExprId,
-}
-
-#[derive(Debug)]
-pub struct FruInfo<'tcx> {
-    pub base: ExprId,
-    pub field_types: Box<[Ty<'tcx>]>,
-}
-
-#[derive(Debug)]
-pub struct Arm<'tcx> {
-    pub pattern: Pat<'tcx>,
-    pub guard: Option<Guard<'tcx>>,
-    pub body: ExprId,
-    pub lint_level: LintLevel,
-    pub scope: region::Scope,
-    pub span: Span,
-}
-
-#[derive(Debug)]
-pub enum Guard<'tcx> {
-    If(ExprId),
-    IfLet(Pat<'tcx>, ExprId),
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum LogicalOp {
-    And,
-    Or,
-}
-
-#[derive(Debug)]
-pub enum InlineAsmOperand<'tcx> {
-    In {
-        reg: InlineAsmRegOrRegClass,
-        expr: ExprId,
-    },
-    Out {
-        reg: InlineAsmRegOrRegClass,
-        late: bool,
-        expr: Option<ExprId>,
-    },
-    InOut {
-        reg: InlineAsmRegOrRegClass,
-        late: bool,
-        expr: ExprId,
-    },
-    SplitInOut {
-        reg: InlineAsmRegOrRegClass,
-        late: bool,
-        in_expr: ExprId,
-        out_expr: Option<ExprId>,
-    },
-    Const {
-        value: &'tcx Const<'tcx>,
-        span: Span,
-    },
-    SymFn {
-        expr: ExprId,
-    },
-    SymStatic {
-        def_id: DefId,
-    },
-}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index e4419070cbd..389a7595315 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -1,8 +1,8 @@
 use super::usefulness::{
-    compute_match_usefulness, expand_pattern, MatchArm, MatchCheckCtxt, Reachability,
+    compute_match_usefulness, expand_pattern, is_wildcard, MatchArm, MatchCheckCtxt, Reachability,
     UsefulnessReport,
 };
-use super::{PatCtxt, PatKind, PatternError};
+use super::{PatCtxt, PatternError};
 
 use rustc_arena::TypedArena;
 use rustc_ast::Mutability;
@@ -12,6 +12,7 @@ use rustc_hir::def::*;
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_hir::{HirId, Pat};
+use rustc_middle::thir::PatKind;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_session::lint::builtin::BINDINGS_WITH_VARIANT_NAME;
 use rustc_session::lint::builtin::{IRREFUTABLE_LET_PATTERNS, UNREACHABLE_PATTERNS};
@@ -344,7 +345,7 @@ fn check_for_bindings_named_same_as_variants(cx: &MatchVisitor<'_, '_>, pat: &Pa
 
 /// Checks for common cases of "catchall" patterns that may not be intended as such.
 fn pat_is_catchall(pat: &super::Pat<'_>) -> bool {
-    use super::PatKind::*;
+    use PatKind::*;
     match &*pat.kind {
         Binding { subpattern: None, .. } => true,
         Binding { subpattern: Some(s), .. } | Deref { subpattern: s } => pat_is_catchall(s),
@@ -514,7 +515,7 @@ fn non_exhaustive_match<'p, 'tcx>(
     if (scrut_ty == cx.tcx.types.usize || scrut_ty == cx.tcx.types.isize)
         && !is_empty_match
         && witnesses.len() == 1
-        && witnesses[0].is_wildcard()
+        && is_wildcard(&witnesses[0])
     {
         err.note(&format!(
             "`{}` does not have a fixed maximum value, \
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index c0624c805a6..369fff00456 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -2,6 +2,7 @@ use rustc_hir as hir;
 use rustc_index::vec::Idx;
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_middle::mir::Field;
+use rustc_middle::thir::{FieldPat, Pat, PatKind};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt};
 use rustc_session::lint;
@@ -12,7 +13,7 @@ use rustc_trait_selection::traits::{self, ObligationCause, PredicateObligation};
 
 use std::cell::Cell;
 
-use super::{FieldPat, Pat, PatCtxt, PatKind};
+use super::PatCtxt;
 
 impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     /// Converts an evaluated constant to a pattern (if possible).
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index db0f487645f..4b5b648c504 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -46,8 +46,7 @@ use self::Constructor::*;
 use self::SliceKind::*;
 
 use super::compare_const_vals;
-use super::usefulness::{MatchCheckCtxt, PatCtxt};
-use super::{FieldPat, Pat, PatKind, PatRange};
+use super::usefulness::{is_wildcard, MatchCheckCtxt, PatCtxt};
 
 use rustc_data_structures::captures::Captures;
 use rustc_index::vec::Idx;
@@ -55,6 +54,7 @@ use rustc_index::vec::Idx;
 use rustc_hir::{HirId, RangeEnd};
 use rustc_middle::mir::interpret::ConstValue;
 use rustc_middle::mir::Field;
+use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{self, Const, Ty, TyCtxt};
 use rustc_session::lint;
@@ -1245,13 +1245,13 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
                         // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
                         // This is incorrect if the size is not known, since `[_, ..]` captures
                         // arrays of lengths `>= 1` whereas `[..]` captures any length.
-                        while !prefix.is_empty() && prefix.last().unwrap().is_wildcard() {
+                        while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
                             prefix.pop();
                         }
                     }
                     let suffix: Vec<_> = if slice.array_len.is_some() {
                         // Same as above.
-                        subpatterns.skip_while(Pat::is_wildcard).collect()
+                        subpatterns.skip_while(is_wildcard).collect()
                     } else {
                         subpatterns.collect()
                     };
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index 9ac79a37ac6..3225d302cb3 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -11,7 +11,7 @@ use crate::thir::util::UserAnnotatedTyHelpers;
 
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
+use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::pat_util::EnumerateAndAdjustIterator;
 use rustc_hir::RangeEnd;
 use rustc_index::vec::Idx;
@@ -19,16 +19,12 @@ use rustc_middle::mir::interpret::{get_slice_bytes, ConstValue};
 use rustc_middle::mir::interpret::{ErrorHandled, LitToConstError, LitToConstInput};
 use rustc_middle::mir::UserTypeProjection;
 use rustc_middle::mir::{BorrowKind, Field, Mutability};
+use rustc_middle::thir::{Ascription, BindingMode, FieldPat, Pat, PatKind, PatRange, PatTyProj};
 use rustc_middle::ty::subst::{GenericArg, SubstsRef};
 use rustc_middle::ty::{self, AdtDef, DefIdTree, Region, Ty, TyCtxt, UserType};
-use rustc_middle::ty::{
-    CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations,
-};
-use rustc_span::{Span, Symbol, DUMMY_SP};
-use rustc_target::abi::VariantIdx;
+use rustc_span::{Span, Symbol};
 
 use std::cmp::Ordering;
-use std::fmt;
 
 #[derive(Clone, Debug)]
 crate enum PatternError {
@@ -39,317 +35,6 @@ crate enum PatternError {
     NonConstPath(Span),
 }
 
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub enum BindingMode {
-    ByValue,
-    ByRef(BorrowKind),
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct FieldPat<'tcx> {
-    pub field: Field,
-    pub pattern: Pat<'tcx>,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct Pat<'tcx> {
-    pub ty: Ty<'tcx>,
-    pub span: Span,
-    pub kind: Box<PatKind<'tcx>>,
-}
-
-impl<'tcx> Pat<'tcx> {
-    pub(crate) fn wildcard_from_ty(ty: Ty<'tcx>) -> Self {
-        Pat { ty, span: DUMMY_SP, kind: Box::new(PatKind::Wild) }
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub struct PatTyProj<'tcx> {
-    pub user_ty: CanonicalUserType<'tcx>,
-}
-
-impl<'tcx> PatTyProj<'tcx> {
-    pub(crate) fn from_user_type(user_annotation: CanonicalUserType<'tcx>) -> Self {
-        Self { user_ty: user_annotation }
-    }
-
-    pub(crate) fn user_ty(
-        self,
-        annotations: &mut CanonicalUserTypeAnnotations<'tcx>,
-        inferred_ty: Ty<'tcx>,
-        span: Span,
-    ) -> UserTypeProjection {
-        UserTypeProjection {
-            base: annotations.push(CanonicalUserTypeAnnotation {
-                span,
-                user_ty: self.user_ty,
-                inferred_ty,
-            }),
-            projs: Vec::new(),
-        }
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub struct Ascription<'tcx> {
-    pub user_ty: PatTyProj<'tcx>,
-    /// Variance to use when relating the type `user_ty` to the **type of the value being
-    /// matched**. Typically, this is `Variance::Covariant`, since the value being matched must
-    /// have a type that is some subtype of the ascribed type.
-    ///
-    /// Note that this variance does not apply for any bindings within subpatterns. The type
-    /// assigned to those bindings must be exactly equal to the `user_ty` given here.
-    ///
-    /// The only place where this field is not `Covariant` is when matching constants, where
-    /// we currently use `Contravariant` -- this is because the constant type just needs to
-    /// be "comparable" to the type of the input value. So, for example:
-    ///
-    /// ```text
-    /// match x { "foo" => .. }
-    /// ```
-    ///
-    /// requires that `&'static str <: T_x`, where `T_x` is the type of `x`. Really, we should
-    /// probably be checking for a `PartialEq` impl instead, but this preserves the behavior
-    /// of the old type-check for now. See #57280 for details.
-    pub variance: ty::Variance,
-    pub user_ty_span: Span,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub enum PatKind<'tcx> {
-    Wild,
-
-    AscribeUserType {
-        ascription: Ascription<'tcx>,
-        subpattern: Pat<'tcx>,
-    },
-
-    /// `x`, `ref x`, `x @ P`, etc.
-    Binding {
-        mutability: Mutability,
-        name: Symbol,
-        mode: BindingMode,
-        var: hir::HirId,
-        ty: Ty<'tcx>,
-        subpattern: Option<Pat<'tcx>>,
-        /// Is this the leftmost occurrence of the binding, i.e., is `var` the
-        /// `HirId` of this pattern?
-        is_primary: bool,
-    },
-
-    /// `Foo(...)` or `Foo{...}` or `Foo`, where `Foo` is a variant name from an ADT with
-    /// multiple variants.
-    Variant {
-        adt_def: &'tcx AdtDef,
-        substs: SubstsRef<'tcx>,
-        variant_index: VariantIdx,
-        subpatterns: Vec<FieldPat<'tcx>>,
-    },
-
-    /// `(...)`, `Foo(...)`, `Foo{...}`, or `Foo`, where `Foo` is a variant name from an ADT with
-    /// a single variant.
-    Leaf {
-        subpatterns: Vec<FieldPat<'tcx>>,
-    },
-
-    /// `box P`, `&P`, `&mut P`, etc.
-    Deref {
-        subpattern: Pat<'tcx>,
-    },
-
-    /// One of the following:
-    /// * `&str`, which will be handled as a string pattern and thus exhaustiveness
-    ///   checking will detect if you use the same string twice in different patterns.
-    /// * integer, bool, char or float, which will be handled by exhaustivenes to cover exactly
-    ///   its own value, similar to `&str`, but these values are much simpler.
-    /// * Opaque constants, that must not be matched structurally. So anything that does not derive
-    ///   `PartialEq` and `Eq`.
-    Constant {
-        value: &'tcx ty::Const<'tcx>,
-    },
-
-    Range(PatRange<'tcx>),
-
-    /// Matches against a slice, checking the length and extracting elements.
-    /// irrefutable when there is a slice pattern and both `prefix` and `suffix` are empty.
-    /// e.g., `&[ref xs @ ..]`.
-    Slice {
-        prefix: Vec<Pat<'tcx>>,
-        slice: Option<Pat<'tcx>>,
-        suffix: Vec<Pat<'tcx>>,
-    },
-
-    /// Fixed match against an array; irrefutable.
-    Array {
-        prefix: Vec<Pat<'tcx>>,
-        slice: Option<Pat<'tcx>>,
-        suffix: Vec<Pat<'tcx>>,
-    },
-
-    /// An or-pattern, e.g. `p | q`.
-    /// Invariant: `pats.len() >= 2`.
-    Or {
-        pats: Vec<Pat<'tcx>>,
-    },
-}
-
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub struct PatRange<'tcx> {
-    pub lo: &'tcx ty::Const<'tcx>,
-    pub hi: &'tcx ty::Const<'tcx>,
-    pub end: RangeEnd,
-}
-
-impl<'tcx> fmt::Display for Pat<'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // Printing lists is a chore.
-        let mut first = true;
-        let mut start_or_continue = |s| {
-            if first {
-                first = false;
-                ""
-            } else {
-                s
-            }
-        };
-        let mut start_or_comma = || start_or_continue(", ");
-
-        match *self.kind {
-            PatKind::Wild => write!(f, "_"),
-            PatKind::AscribeUserType { ref subpattern, .. } => write!(f, "{}: _", subpattern),
-            PatKind::Binding { mutability, name, mode, ref subpattern, .. } => {
-                let is_mut = match mode {
-                    BindingMode::ByValue => mutability == Mutability::Mut,
-                    BindingMode::ByRef(bk) => {
-                        write!(f, "ref ")?;
-                        matches!(bk, BorrowKind::Mut { .. })
-                    }
-                };
-                if is_mut {
-                    write!(f, "mut ")?;
-                }
-                write!(f, "{}", name)?;
-                if let Some(ref subpattern) = *subpattern {
-                    write!(f, " @ {}", subpattern)?;
-                }
-                Ok(())
-            }
-            PatKind::Variant { ref subpatterns, .. } | PatKind::Leaf { ref subpatterns } => {
-                let variant = match *self.kind {
-                    PatKind::Variant { adt_def, variant_index, .. } => {
-                        Some(&adt_def.variants[variant_index])
-                    }
-                    _ => {
-                        if let ty::Adt(adt, _) = self.ty.kind() {
-                            if !adt.is_enum() {
-                                Some(&adt.variants[VariantIdx::new(0)])
-                            } else {
-                                None
-                            }
-                        } else {
-                            None
-                        }
-                    }
-                };
-
-                if let Some(variant) = variant {
-                    write!(f, "{}", variant.ident)?;
-
-                    // Only for Adt we can have `S {...}`,
-                    // which we handle separately here.
-                    if variant.ctor_kind == CtorKind::Fictive {
-                        write!(f, " {{ ")?;
-
-                        let mut printed = 0;
-                        for p in subpatterns {
-                            if let PatKind::Wild = *p.pattern.kind {
-                                continue;
-                            }
-                            let name = variant.fields[p.field.index()].ident;
-                            write!(f, "{}{}: {}", start_or_comma(), name, p.pattern)?;
-                            printed += 1;
-                        }
-
-                        if printed < variant.fields.len() {
-                            write!(f, "{}..", start_or_comma())?;
-                        }
-
-                        return write!(f, " }}");
-                    }
-                }
-
-                let num_fields = variant.map_or(subpatterns.len(), |v| v.fields.len());
-                if num_fields != 0 || variant.is_none() {
-                    write!(f, "(")?;
-                    for i in 0..num_fields {
-                        write!(f, "{}", start_or_comma())?;
-
-                        // Common case: the field is where we expect it.
-                        if let Some(p) = subpatterns.get(i) {
-                            if p.field.index() == i {
-                                write!(f, "{}", p.pattern)?;
-                                continue;
-                            }
-                        }
-
-                        // Otherwise, we have to go looking for it.
-                        if let Some(p) = subpatterns.iter().find(|p| p.field.index() == i) {
-                            write!(f, "{}", p.pattern)?;
-                        } else {
-                            write!(f, "_")?;
-                        }
-                    }
-                    write!(f, ")")?;
-                }
-
-                Ok(())
-            }
-            PatKind::Deref { ref subpattern } => {
-                match self.ty.kind() {
-                    ty::Adt(def, _) if def.is_box() => write!(f, "box ")?,
-                    ty::Ref(_, _, mutbl) => {
-                        write!(f, "&{}", mutbl.prefix_str())?;
-                    }
-                    _ => bug!("{} is a bad Deref pattern type", self.ty),
-                }
-                write!(f, "{}", subpattern)
-            }
-            PatKind::Constant { value } => write!(f, "{}", value),
-            PatKind::Range(PatRange { lo, hi, end }) => {
-                write!(f, "{}", lo)?;
-                write!(f, "{}", end)?;
-                write!(f, "{}", hi)
-            }
-            PatKind::Slice { ref prefix, ref slice, ref suffix }
-            | PatKind::Array { ref prefix, ref slice, ref suffix } => {
-                write!(f, "[")?;
-                for p in prefix {
-                    write!(f, "{}{}", start_or_comma(), p)?;
-                }
-                if let Some(ref slice) = *slice {
-                    write!(f, "{}", start_or_comma())?;
-                    match *slice.kind {
-                        PatKind::Wild => {}
-                        _ => write!(f, "{}", slice)?,
-                    }
-                    write!(f, "..")?;
-                }
-                for p in suffix {
-                    write!(f, "{}{}", start_or_comma(), p)?;
-                }
-                write!(f, "]")
-            }
-            PatKind::Or { ref pats } => {
-                for pat in pats {
-                    write!(f, "{}{}", start_or_continue(" | "), pat)?;
-                }
-                Ok(())
-            }
-        }
-    }
-}
-
 crate struct PatCtxt<'a, 'tcx> {
     crate tcx: TyCtxt<'tcx>,
     crate param_env: ty::ParamEnv<'tcx>,
@@ -358,22 +43,20 @@ crate struct PatCtxt<'a, 'tcx> {
     include_lint_checks: bool,
 }
 
-impl<'a, 'tcx> Pat<'tcx> {
-    crate fn from_hir(
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-        typeck_results: &'a ty::TypeckResults<'tcx>,
-        pat: &'tcx hir::Pat<'tcx>,
-    ) -> Self {
-        let mut pcx = PatCtxt::new(tcx, param_env, typeck_results);
-        let result = pcx.lower_pattern(pat);
-        if !pcx.errors.is_empty() {
-            let msg = format!("encountered errors lowering pattern: {:?}", pcx.errors);
-            tcx.sess.delay_span_bug(pat.span, &msg);
-        }
-        debug!("Pat::from_hir({:?}) = {:?}", pat, result);
-        result
-    }
+crate fn pat_from_hir<'a, 'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    typeck_results: &'a ty::TypeckResults<'tcx>,
+    pat: &'tcx hir::Pat<'tcx>,
+) -> Pat<'tcx> {
+    let mut pcx = PatCtxt::new(tcx, param_env, typeck_results);
+    let result = pcx.lower_pattern(pat);
+    if !pcx.errors.is_empty() {
+        let msg = format!("encountered errors lowering pattern: {:?}", pcx.errors);
+        tcx.sess.delay_span_bug(pat.span, &msg);
+    }
+    debug!("pat_from_hir({:?}) = {:?}", pat, result);
+    result
 }
 
 impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index dce0df8473b..5d4eb75155a 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -284,7 +284,6 @@ use self::Usefulness::*;
 use self::WitnessPreference::*;
 
 use super::deconstruct_pat::{Constructor, Fields, SplitWildcard};
-use super::{Pat, PatKind};
 use super::{PatternFoldable, PatternFolder};
 
 use rustc_data_structures::captures::Captures;
@@ -293,6 +292,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_arena::TypedArena;
 use rustc_hir::def_id::DefId;
 use rustc_hir::HirId;
+use rustc_middle::thir::{Pat, PatKind};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::Span;
 
@@ -382,31 +382,29 @@ impl<'tcx> PatternFolder<'tcx> for LiteralExpander {
     }
 }
 
-impl<'tcx> Pat<'tcx> {
-    pub(super) fn is_wildcard(&self) -> bool {
-        matches!(*self.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild)
-    }
+pub(super) fn is_wildcard(pat: &Pat<'_>) -> bool {
+    matches!(*pat.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild)
+}
 
-    fn is_or_pat(&self) -> bool {
-        matches!(*self.kind, PatKind::Or { .. })
-    }
+fn is_or_pat(pat: &Pat<'_>) -> bool {
+    matches!(*pat.kind, PatKind::Or { .. })
+}
 
-    /// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
-    fn expand_or_pat(&self) -> Vec<&Self> {
-        fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
-            if let PatKind::Or { pats } = pat.kind.as_ref() {
-                for pat in pats {
-                    expand(pat, vec);
-                }
-            } else {
-                vec.push(pat)
+/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
+fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
+    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
+        if let PatKind::Or { pats } = pat.kind.as_ref() {
+            for pat in pats {
+                expand(pat, vec);
             }
+        } else {
+            vec.push(pat)
         }
-
-        let mut pats = Vec::new();
-        expand(self, &mut pats);
-        pats
     }
+
+    let mut pats = Vec::new();
+    expand(pat, &mut pats);
+    pats
 }
 
 /// A row of a matrix. Rows of len 1 are very common, which is why `SmallVec[_; 2]`
@@ -451,7 +449,7 @@ impl<'p, 'tcx> PatStack<'p, 'tcx> {
     // Recursively expand the first pattern into its subpatterns. Only useful if the pattern is an
     // or-pattern. Panics if `self` is empty.
     fn expand_or_pat<'a>(&'a self) -> impl Iterator<Item = PatStack<'p, 'tcx>> + Captures<'a> {
-        self.head().expand_or_pat().into_iter().map(move |pat| {
+        expand_or_pat(self.head()).into_iter().map(move |pat| {
             let mut new_patstack = PatStack::from_pattern(pat);
             new_patstack.pats.extend_from_slice(&self.pats[1..]);
             new_patstack
@@ -525,7 +523,7 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> {
     /// Pushes a new row to the matrix. If the row starts with an or-pattern, this recursively
     /// expands it.
     fn push(&mut self, row: PatStack<'p, 'tcx>) {
-        if !row.is_empty() && row.head().is_or_pat() {
+        if !row.is_empty() && is_or_pat(row.head()) {
             for row in row.expand_or_pat() {
                 self.patterns.push(row);
             }
@@ -760,7 +758,7 @@ impl<'p, 'tcx> SubPatSet<'p, 'tcx> {
                     }
                 }
                 SubPatSet::Alt { subpats, pat, alt_count, .. } => {
-                    let expanded = pat.expand_or_pat();
+                    let expanded = expand_or_pat(pat);
                     for i in 0..*alt_count {
                         let sub_set = subpats.get(&i).unwrap_or(&SubPatSet::Empty);
                         if sub_set.is_empty() {
@@ -1118,7 +1116,7 @@ fn is_useful<'p, 'tcx>(
     let pcx = PatCtxt { cx, ty, span: v.head().span, is_top_level };
 
     // If the first pattern is an or-pattern, expand it.
-    let ret = if v.head().is_or_pat() {
+    let ret = if is_or_pat(v.head()) {
         debug!("expanding or-pattern");
         let v_head = v.head();
         let vs: Vec<_> = v.expand_or_pat().collect();
@@ -1174,7 +1172,7 @@ fn is_useful<'p, 'tcx>(
 #[derive(Clone, Copy)]
 crate struct MatchArm<'p, 'tcx> {
     /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`.
-    crate pat: &'p super::Pat<'tcx>,
+    crate pat: &'p Pat<'tcx>,
     crate hir_id: HirId,
     crate has_guard: bool,
 }
@@ -1196,7 +1194,7 @@ crate struct UsefulnessReport<'p, 'tcx> {
     crate arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Reachability)>,
     /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of
     /// exhaustiveness.
-    crate non_exhaustiveness_witnesses: Vec<super::Pat<'tcx>>,
+    crate non_exhaustiveness_witnesses: Vec<Pat<'tcx>>,
 }
 
 /// The entrypoint for the usefulness algorithm. Computes whether a match is exhaustive and which
@@ -1232,7 +1230,7 @@ crate fn compute_match_usefulness<'p, 'tcx>(
         })
         .collect();
 
-    let wild_pattern = cx.pattern_arena.alloc(super::Pat::wildcard_from_ty(scrut_ty));
+    let wild_pattern = cx.pattern_arena.alloc(Pat::wildcard_from_ty(scrut_ty));
     let v = PatStack::from_pattern(wild_pattern);
     let usefulness = is_useful(cx, &matrix, &v, ConstructWitness, scrut_hir_id, false, true);
     let non_exhaustiveness_witnesses = match usefulness {
diff --git a/compiler/rustc_mir_build/src/thir/visit.rs b/compiler/rustc_mir_build/src/thir/visit.rs
index 671d1fe9b03..1a60b1de7fd 100644
--- a/compiler/rustc_mir_build/src/thir/visit.rs
+++ b/compiler/rustc_mir_build/src/thir/visit.rs
@@ -1,4 +1,5 @@
-use crate::thir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::Const;
 
 pub trait Visitor<'a, 'tcx: 'a>: Sized {
     fn thir(&self) -> &'a Thir<'tcx>;
diff --git a/src/test/ui/async-await/async-unsafe-fn-call-in-safe.rs b/src/test/ui/async-await/async-unsafe-fn-call-in-safe.rs
index 2ed343b4a07..fc37822cb7b 100644
--- a/src/test/ui/async-await/async-unsafe-fn-call-in-safe.rs
+++ b/src/test/ui/async-await/async-unsafe-fn-call-in-safe.rs
@@ -16,6 +16,6 @@ async fn g() {
 }
 
 fn main() {
-    S::f(); //~ ERROR call to unsafe function is unsafe
-    f(); //~ ERROR call to unsafe function is unsafe
+    S::f(); //[mir]~ ERROR call to unsafe function is unsafe
+    f(); //[mir]~ ERROR call to unsafe function is unsafe
 }
diff --git a/src/test/ui/async-await/async-unsafe-fn-call-in-safe.thir.stderr b/src/test/ui/async-await/async-unsafe-fn-call-in-safe.thir.stderr
index d22413beecb..21ba45d7f1e 100644
--- a/src/test/ui/async-await/async-unsafe-fn-call-in-safe.thir.stderr
+++ b/src/test/ui/async-await/async-unsafe-fn-call-in-safe.thir.stderr
@@ -14,22 +14,6 @@ LL |     f();
    |
    = note: consult the function's documentation for information on how to avoid undefined behavior
 
-error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
-  --> $DIR/async-unsafe-fn-call-in-safe.rs:19:5
-   |
-LL |     S::f();
-   |     ^^^^^^ call to unsafe function
-   |
-   = note: consult the function's documentation for information on how to avoid undefined behavior
-
-error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
-  --> $DIR/async-unsafe-fn-call-in-safe.rs:20:5
-   |
-LL |     f();
-   |     ^^^ call to unsafe function
-   |
-   = note: consult the function's documentation for information on how to avoid undefined behavior
-
-error: aborting due to 4 previous errors
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0133`.
diff --git a/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.rs b/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.rs
index 1ce78147970..031e67a1e3c 100644
--- a/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.rs
+++ b/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.rs
@@ -9,5 +9,5 @@ fn main() {
     let a: [u8; foo()];
     //~^ ERROR call to unsafe function is unsafe and requires unsafe function or block
     foo();
-    //~^ ERROR call to unsafe function is unsafe and requires unsafe function or block
+    //[mir]~^ ERROR call to unsafe function is unsafe and requires unsafe function or block
 }
diff --git a/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.thir.stderr b/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.thir.stderr
index b643ecc0ce8..c6077da768b 100644
--- a/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.thir.stderr
+++ b/src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.thir.stderr
@@ -1,12 +1,4 @@
 error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
-  --> $DIR/const-extern-fn-requires-unsafe.rs:11:5
-   |
-LL |     foo();
-   |     ^^^^^ call to unsafe function
-   |
-   = note: consult the function's documentation for information on how to avoid undefined behavior
-
-error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
   --> $DIR/const-extern-fn-requires-unsafe.rs:9:17
    |
 LL |     let a: [u8; foo()];
@@ -14,6 +6,6 @@ LL |     let a: [u8; foo()];
    |
    = note: consult the function's documentation for information on how to avoid undefined behavior
 
-error: aborting due to 2 previous errors
+error: aborting due to previous error
 
 For more information about this error, try `rustc --explain E0133`.
diff --git a/src/test/ui/feature-gates/feature-gate-const_fn_transmute.thir.stderr b/src/test/ui/feature-gates/feature-gate-const_fn_transmute.thir.stderr
index 04efea0b230..df0de7a9590 100644
--- a/src/test/ui/feature-gates/feature-gate-const_fn_transmute.thir.stderr
+++ b/src/test/ui/feature-gates/feature-gate-const_fn_transmute.thir.stderr
@@ -58,6 +58,14 @@ LL | const unsafe fn unsafe_transmute_fn_core_intrinsic() -> u32 { core::intrins
    = help: add `#![feature(const_fn_transmute)]` to the crate attributes to enable
    = note: `transmute` is only allowed in constants and statics for now
 
+error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
+  --> $DIR/feature-gate-const_fn_transmute.rs:29:39
+   |
+LL | const fn safe_transmute_fn() -> u32 { mem::transmute(Foo(3)) }
+   |                                       ^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
+   |
+   = note: consult the function's documentation for information on how to avoid undefined behavior
+
 error[E0658]: `transmute` is not allowed in constant functions
   --> $DIR/feature-gate-const_fn_transmute.rs:29:39
    |
@@ -68,49 +76,41 @@ LL | const fn safe_transmute_fn() -> u32 { mem::transmute(Foo(3)) }
    = help: add `#![feature(const_fn_transmute)]` to the crate attributes to enable
    = note: `transmute` is only allowed in constants and statics for now
 
-error[E0658]: `transmute` is not allowed in constant functions
+error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
   --> $DIR/feature-gate-const_fn_transmute.rs:33:49
    |
 LL | const fn safe_transmute_fn_intrinsic() -> u32 { std::intrinsics::transmute(Foo(3)) }
-   |                                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |                                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
    |
-   = note: see issue #53605 <https://github.com/rust-lang/rust/issues/53605> for more information
-   = help: add `#![feature(const_fn_transmute)]` to the crate attributes to enable
-   = note: `transmute` is only allowed in constants and statics for now
+   = note: consult the function's documentation for information on how to avoid undefined behavior
 
 error[E0658]: `transmute` is not allowed in constant functions
-  --> $DIR/feature-gate-const_fn_transmute.rs:37:54
+  --> $DIR/feature-gate-const_fn_transmute.rs:33:49
    |
-LL | const fn safe_transmute_fn_core_intrinsic() -> u32 { core::intrinsics::transmute(Foo(3)) }
-   |                                                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | const fn safe_transmute_fn_intrinsic() -> u32 { std::intrinsics::transmute(Foo(3)) }
+   |                                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = note: see issue #53605 <https://github.com/rust-lang/rust/issues/53605> for more information
    = help: add `#![feature(const_fn_transmute)]` to the crate attributes to enable
    = note: `transmute` is only allowed in constants and statics for now
 
 error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
-  --> $DIR/feature-gate-const_fn_transmute.rs:29:39
-   |
-LL | const fn safe_transmute_fn() -> u32 { mem::transmute(Foo(3)) }
-   |                                       ^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
-   |
-   = note: consult the function's documentation for information on how to avoid undefined behavior
-
-error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
-  --> $DIR/feature-gate-const_fn_transmute.rs:33:49
+  --> $DIR/feature-gate-const_fn_transmute.rs:37:54
    |
-LL | const fn safe_transmute_fn_intrinsic() -> u32 { std::intrinsics::transmute(Foo(3)) }
-   |                                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
+LL | const fn safe_transmute_fn_core_intrinsic() -> u32 { core::intrinsics::transmute(Foo(3)) }
+   |                                                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
    |
    = note: consult the function's documentation for information on how to avoid undefined behavior
 
-error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
+error[E0658]: `transmute` is not allowed in constant functions
   --> $DIR/feature-gate-const_fn_transmute.rs:37:54
    |
 LL | const fn safe_transmute_fn_core_intrinsic() -> u32 { core::intrinsics::transmute(Foo(3)) }
-   |                                                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
+   |                                                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-   = note: consult the function's documentation for information on how to avoid undefined behavior
+   = note: see issue #53605 <https://github.com/rust-lang/rust/issues/53605> for more information
+   = help: add `#![feature(const_fn_transmute)]` to the crate attributes to enable
+   = note: `transmute` is only allowed in constants and statics for now
 
 error: aborting due to 12 previous errors
 
diff --git a/src/test/ui/issues/issue-16538.thir.stderr b/src/test/ui/issues/issue-16538.thir.stderr
index d7e8c08bb01..435334c3228 100644
--- a/src/test/ui/issues/issue-16538.thir.stderr
+++ b/src/test/ui/issues/issue-16538.thir.stderr
@@ -1,3 +1,11 @@
+error[E0133]: use of extern static is unsafe and requires unsafe function or block
+  --> $DIR/issue-16538.rs:14:34
+   |
+LL | static foo: *const Y::X = Y::foo(Y::x as *const Y::X);
+   |                                  ^^^^ use of extern static
+   |
+   = note: extern statics are not controlled by the Rust type system: invalid data, aliasing violations or data races will cause undefined behavior
+
 error[E0015]: calls in statics are limited to constant functions, tuple structs and tuple variants
   --> $DIR/issue-16538.rs:14:27
    |
@@ -13,14 +21,6 @@ LL | static foo: *const Y::X = Y::foo(Y::x as *const Y::X);
    = help: the trait `Sync` is not implemented for `*const usize`
    = note: shared static variables must have a type that implements `Sync`
 
-error[E0133]: use of extern static is unsafe and requires unsafe function or block
-  --> $DIR/issue-16538.rs:14:34
-   |
-LL | static foo: *const Y::X = Y::foo(Y::x as *const Y::X);
-   |                                  ^^^^ use of extern static
-   |
-   = note: extern statics are not controlled by the Rust type system: invalid data, aliasing violations or data races will cause undefined behavior
-
 error: aborting due to 3 previous errors
 
 Some errors have detailed explanations: E0015, E0133, E0277.