Merge remote-tracking branch 'upstream/master' into impl-16351-nightly

Signed-off-by: Sean Cross <sean@xobs.io>

756 files changed, 21316 insertions, 17072 deletions

diff --git a/compiler/rustc/Cargo.toml b/compiler/rustc/Cargo.toml
index 6e6c0c71a1f..ca6055c46a6 100644
--- a/compiler/rustc/Cargo.toml
+++ b/compiler/rustc/Cargo.toml
@@ -11,12 +11,16 @@ rustc_driver = { path = "../rustc_driver" }
 # crate is intended to be used by codegen backends, which may not be in-tree.
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 
-[dependencies.jemalloc-sys]
-version = '0.3.0'
+[dependencies.tikv-jemalloc-sys]
+version = '0.4.0'
 optional = true
 features = ['unprefixed_malloc_on_supported_platforms']
 
+[dependencies.tikv-jemallocator]
+version = '0.4.0'
+optional = true
+
 [features]
-jemalloc = ['jemalloc-sys']
+jemalloc = ['tikv-jemalloc-sys', 'tikv-jemallocator']
 llvm = ['rustc_driver/llvm']
 max_level_info = ['rustc_driver/max_level_info']
diff --git a/compiler/rustc/src/main.rs b/compiler/rustc/src/main.rs
index 6bc5aa6382c..c80fab99496 100644
--- a/compiler/rustc/src/main.rs
+++ b/compiler/rustc/src/main.rs
@@ -1,13 +1,26 @@
+// Configure jemalloc as the `global_allocator` when configured. This is
+// so that we use the sized deallocation apis jemalloc provides
+// (namely `sdallocx`).
+//
+// The symbol overrides documented below are also performed so that we can
+// ensure that we use a consistent allocator across the rustc <-> llvm boundary
+#[cfg(feature = "jemalloc")]
+#[global_allocator]
+static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+#[cfg(feature = "tikv-jemalloc-sys")]
+use tikv_jemalloc_sys as jemalloc_sys;
+
 fn main() {
     // Pull in jemalloc when enabled.
     //
     // Note that we're pulling in a static copy of jemalloc which means that to
     // pull it in we need to actually reference its symbols for it to get
     // linked. The two crates we link to here, std and rustc_driver, are both
-    // dynamic libraries. That means to pull in jemalloc we need to actually
+    // dynamic libraries. That means to pull in jemalloc we actually need to
     // reference allocation symbols one way or another (as this file is the only
     // object code in the rustc executable).
-    #[cfg(feature = "jemalloc-sys")]
+    #[cfg(feature = "tikv-jemalloc-sys")]
     {
         use std::os::raw::{c_int, c_void};
 
@@ -24,6 +37,20 @@ fn main() {
         static _F5: unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void = jemalloc_sys::realloc;
         #[used]
         static _F6: unsafe extern "C" fn(*mut c_void) = jemalloc_sys::free;
+
+        // On OSX, jemalloc doesn't directly override malloc/free, but instead
+        // registers itself with the allocator's zone APIs in a ctor. However,
+        // the linker doesn't seem to consider ctors as "used" when statically
+        // linking, so we need to explicitly depend on the function.
+        #[cfg(target_os = "macos")]
+        {
+            extern "C" {
+                fn _rjem_je_zone_register();
+            }
+
+            #[used]
+            static _F7: unsafe extern "C" fn() = _rjem_je_zone_register;
+        }
     }
 
     rustc_driver::set_sigpipe_handler();
diff --git a/compiler/rustc_apfloat/src/ieee.rs b/compiler/rustc_apfloat/src/ieee.rs
index 71bcb8f090d..96277950cfe 100644
--- a/compiler/rustc_apfloat/src/ieee.rs
+++ b/compiler/rustc_apfloat/src/ieee.rs
@@ -2273,6 +2273,7 @@ impl Loss {
 mod sig {
     use super::{limbs_for_bits, ExpInt, Limb, Loss, LIMB_BITS};
     use core::cmp::Ordering;
+    use core::iter;
     use core::mem;
 
     pub(super) fn is_all_zeros(limbs: &[Limb]) -> bool {
@@ -2483,7 +2484,7 @@ mod sig {
     pub(super) fn add(a: &mut [Limb], b: &[Limb], mut c: Limb) -> Limb {
         assert!(c <= 1);
 
-        for (a, &b) in a.iter_mut().zip(b) {
+        for (a, &b) in iter::zip(a, b) {
             let (r, overflow) = a.overflowing_add(b);
             let (r, overflow2) = r.overflowing_add(c);
             *a = r;
@@ -2497,7 +2498,7 @@ mod sig {
     pub(super) fn sub(a: &mut [Limb], b: &[Limb], mut c: Limb) -> Limb {
         assert!(c <= 1);
 
-        for (a, &b) in a.iter_mut().zip(b) {
+        for (a, &b) in iter::zip(a, b) {
             let (r, overflow) = a.overflowing_sub(b);
             let (r, overflow2) = r.overflowing_sub(c);
             *a = r;
diff --git a/compiler/rustc_apfloat/src/lib.rs b/compiler/rustc_apfloat/src/lib.rs
index 4a845fcb691..c648147d108 100644
--- a/compiler/rustc_apfloat/src/lib.rs
+++ b/compiler/rustc_apfloat/src/lib.rs
@@ -33,8 +33,9 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![no_std]
 #![forbid(unsafe_code)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 
 #[macro_use]
 extern crate alloc;
diff --git a/compiler/rustc_arena/Cargo.toml b/compiler/rustc_arena/Cargo.toml
index f2d039c82ab..5d4d47527d3 100644
--- a/compiler/rustc_arena/Cargo.toml
+++ b/compiler/rustc_arena/Cargo.toml
@@ -5,4 +5,5 @@ version = "0.0.0"
 edition = "2018"
 
 [dependencies]
+rustc_data_structures = { path = "../rustc_data_structures" }
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs
index 721cfdd4459..c3e4945c446 100644
--- a/compiler/rustc_arena/src/lib.rs
+++ b/compiler/rustc_arena/src/lib.rs
@@ -17,6 +17,7 @@
 #![feature(min_specialization)]
 #![cfg_attr(test, feature(test))]
 
+use rustc_data_structures::sync;
 use smallvec::SmallVec;
 
 use std::alloc::Layout;
@@ -235,26 +236,6 @@ impl<T> TypedArena<T> {
         start_ptr
     }
 
-    /// Allocates a slice of objects that are copied into the `TypedArena`, returning a mutable
-    /// reference to it. Will panic if passed a zero-sized types.
-    ///
-    /// Panics:
-    ///
-    ///  - Zero-sized types
-    ///  - Zero-length slices
-    #[inline]
-    pub fn alloc_slice(&self, slice: &[T]) -> &mut [T]
-    where
-        T: Copy,
-    {
-        unsafe {
-            let len = slice.len();
-            let start_ptr = self.alloc_raw_slice(len);
-            slice.as_ptr().copy_to_nonoverlapping(start_ptr, len);
-            slice::from_raw_parts_mut(start_ptr, len)
-        }
-    }
-
     #[inline]
     pub fn alloc_from_iter<I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
         assert!(mem::size_of::<T>() != 0);
@@ -297,22 +278,6 @@ impl<T> TypedArena<T> {
         }
     }
 
-    /// Clears the arena. Deallocates all but the longest chunk which may be reused.
-    pub fn clear(&mut self) {
-        unsafe {
-            // Clear the last chunk, which is partially filled.
-            let mut chunks_borrow = self.chunks.borrow_mut();
-            if let Some(mut last_chunk) = chunks_borrow.last_mut() {
-                self.clear_last_chunk(&mut last_chunk);
-                let len = chunks_borrow.len();
-                // If `T` is ZST, code below has no effect.
-                for mut chunk in chunks_borrow.drain(..len - 1) {
-                    chunk.destroy(chunk.entries);
-                }
-            }
-        }
-    }
-
     // Drops the contents of the last chunk. The last chunk is partially empty, unlike all other
     // chunks.
     fn clear_last_chunk(&self, last_chunk: &mut TypedArenaChunk<T>) {
@@ -556,8 +521,19 @@ struct DropType {
     obj: *mut u8,
 }
 
-unsafe fn drop_for_type<T>(to_drop: *mut u8) {
-    std::ptr::drop_in_place(to_drop as *mut T)
+// SAFETY: we require `T: Send` before type-erasing into `DropType`.
+#[cfg(parallel_compiler)]
+unsafe impl sync::Send for DropType {}
+
+impl DropType {
+    #[inline]
+    unsafe fn new<T: sync::Send>(obj: *mut T) -> Self {
+        unsafe fn drop_for_type<T>(to_drop: *mut u8) {
+            std::ptr::drop_in_place(to_drop as *mut T)
+        }
+
+        DropType { drop_fn: drop_for_type::<T>, obj: obj as *mut u8 }
+    }
 }
 
 impl Drop for DropType {
@@ -585,21 +561,26 @@ pub struct DropArena {
 
 impl DropArena {
     #[inline]
-    pub unsafe fn alloc<T>(&self, object: T) -> &mut T {
+    pub unsafe fn alloc<T>(&self, object: T) -> &mut T
+    where
+        T: sync::Send,
+    {
         let mem = self.arena.alloc_raw(Layout::new::<T>()) as *mut T;
         // Write into uninitialized memory.
         ptr::write(mem, object);
         let result = &mut *mem;
         // Record the destructor after doing the allocation as that may panic
         // and would cause `object`'s destructor to run twice if it was recorded before.
-        self.destructors
-            .borrow_mut()
-            .push(DropType { drop_fn: drop_for_type::<T>, obj: result as *mut T as *mut u8 });
+        self.destructors.borrow_mut().push(DropType::new(result));
         result
     }
 
     #[inline]
-    pub unsafe fn alloc_from_iter<T, I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
+    pub unsafe fn alloc_from_iter<T, I>(&self, iter: I) -> &mut [T]
+    where
+        T: sync::Send,
+        I: IntoIterator<Item = T>,
+    {
         let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect();
         if vec.is_empty() {
             return &mut [];
@@ -620,8 +601,7 @@ impl DropArena {
         // Record the destructors after doing the allocation as that may panic
         // and would cause `object`'s destructor to run twice if it was recorded before.
         for i in 0..len {
-            destructors
-                .push(DropType { drop_fn: drop_for_type::<T>, obj: start_ptr.add(i) as *mut u8 });
+            destructors.push(DropType::new(start_ptr.add(i)));
         }
 
         slice::from_raw_parts_mut(start_ptr, len)
diff --git a/compiler/rustc_arena/src/tests.rs b/compiler/rustc_arena/src/tests.rs
index e8a1f2db1a1..911e577c1ed 100644
--- a/compiler/rustc_arena/src/tests.rs
+++ b/compiler/rustc_arena/src/tests.rs
@@ -11,6 +11,24 @@ struct Point {
     z: i32,
 }
 
+impl<T> TypedArena<T> {
+    /// Clears the arena. Deallocates all but the longest chunk which may be reused.
+    fn clear(&mut self) {
+        unsafe {
+            // Clear the last chunk, which is partially filled.
+            let mut chunks_borrow = self.chunks.borrow_mut();
+            if let Some(mut last_chunk) = chunks_borrow.last_mut() {
+                self.clear_last_chunk(&mut last_chunk);
+                let len = chunks_borrow.len();
+                // If `T` is ZST, code below has no effect.
+                for mut chunk in chunks_borrow.drain(..len - 1) {
+                    chunk.destroy(chunk.entries);
+                }
+            }
+        }
+    }
+}
+
 #[test]
 pub fn test_unused() {
     let arena: TypedArena<Point> = TypedArena::default();
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index de44a2031ab..e7f19f06ebe 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -100,6 +100,7 @@ pub struct Path {
 }
 
 impl PartialEq<Symbol> for Path {
+    #[inline]
     fn eq(&self, symbol: &Symbol) -> bool {
         self.segments.len() == 1 && { self.segments[0].ident.name == *symbol }
     }
@@ -149,9 +150,17 @@ impl PathSegment {
     pub fn from_ident(ident: Ident) -> Self {
         PathSegment { ident, id: DUMMY_NODE_ID, args: None }
     }
+
     pub fn path_root(span: Span) -> Self {
         PathSegment::from_ident(Ident::new(kw::PathRoot, span))
     }
+
+    pub fn span(&self) -> Span {
+        match &self.args {
+            Some(args) => self.ident.span.to(args.span()),
+            None => self.ident.span,
+        }
+    }
 }
 
 /// The arguments of a path segment.
@@ -647,7 +656,7 @@ impl Pat {
 /// are treated the same as `x: x, y: ref y, z: ref mut z`,
 /// except when `is_shorthand` is true.
 #[derive(Clone, Encodable, Decodable, Debug)]
-pub struct FieldPat {
+pub struct PatField {
     /// The identifier for the field.
     pub ident: Ident,
     /// The pattern the field is destructured to.
@@ -692,7 +701,7 @@ pub enum PatKind {
 
     /// A struct or struct variant pattern (e.g., `Variant {x, y, ..}`).
     /// The `bool` is `true` in the presence of a `..`.
-    Struct(Path, Vec<FieldPat>, /* recovered */ bool),
+    Struct(Path, Vec<PatField>, /* recovered */ bool),
 
     /// A tuple struct/variant pattern (`Variant(x, y, .., z)`).
     TupleStruct(Path, Vec<P<Pat>>),
@@ -754,14 +763,6 @@ pub enum Mutability {
 }
 
 impl Mutability {
-    /// Returns `MutMutable` only if both `self` and `other` are mutable.
-    pub fn and(self, other: Self) -> Self {
-        match self {
-            Mutability::Mut => other,
-            Mutability::Not => Mutability::Not,
-        }
-    }
-
     pub fn invert(self) -> Self {
         match self {
             Mutability::Mut => Mutability::Not,
@@ -915,16 +916,6 @@ impl Stmt {
         }
     }
 
-    pub fn tokens_mut(&mut self) -> Option<&mut LazyTokenStream> {
-        match self.kind {
-            StmtKind::Local(ref mut local) => local.tokens.as_mut(),
-            StmtKind::Item(ref mut item) => item.tokens.as_mut(),
-            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens.as_mut(),
-            StmtKind::Empty => None,
-            StmtKind::MacCall(ref mut mac) => mac.tokens.as_mut(),
-        }
-    }
-
     pub fn has_trailing_semicolon(&self) -> bool {
         match &self.kind {
             StmtKind::Semi(_) => true,
@@ -1037,9 +1028,9 @@ pub struct Arm {
     pub is_placeholder: bool,
 }
 
-/// Access of a named (e.g., `obj.foo`) or unnamed (e.g., `obj.0`) struct field.
+/// A single field in a struct expression, e.g. `x: value` and `y` in `Foo { x: value, y }`.
 #[derive(Clone, Encodable, Decodable, Debug)]
-pub struct Field {
+pub struct ExprField {
     pub attrs: AttrVec,
     pub id: NodeId,
     pub span: Span,
@@ -1083,8 +1074,8 @@ pub struct Expr {
 }
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Expr, 120);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+rustc_data_structures::static_assert_size!(Expr, 104);
 
 impl Expr {
     /// Returns `true` if this expression would be valid somewhere that expects a value;
@@ -1139,6 +1130,14 @@ impl Expr {
         }
     }
 
+    pub fn peel_parens(&self) -> &Expr {
+        let mut expr = self;
+        while let ExprKind::Paren(inner) = &expr.kind {
+            expr = &inner;
+        }
+        expr
+    }
+
     /// Attempts to reparse as `Ty` (for diagnostic purposes).
     pub fn to_ty(&self) -> Option<P<Ty>> {
         let kind = match &self.kind {
@@ -1247,6 +1246,13 @@ pub enum StructRest {
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
+pub struct StructExpr {
+    pub path: Path,
+    pub fields: Vec<ExprField>,
+    pub rest: StructRest,
+}
+
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub enum ExprKind {
     /// A `box x` expression.
     Box(P<Expr>),
@@ -1340,7 +1346,7 @@ pub enum ExprKind {
     Field(P<Expr>, Ident),
     /// An indexing operation (e.g., `foo[2]`).
     Index(P<Expr>, P<Expr>),
-    /// A range (e.g., `1..2`, `1..`, `..2`, `1..=2`, `..=2`; and `..` in destructuring assingment).
+    /// A range (e.g., `1..2`, `1..`, `..2`, `1..=2`, `..=2`; and `..` in destructuring assignment).
     Range(Option<P<Expr>>, Option<P<Expr>>, RangeLimits),
     /// An underscore, used in destructuring assignment to ignore a value.
     Underscore,
@@ -1371,7 +1377,7 @@ pub enum ExprKind {
     /// A struct literal expression.
     ///
     /// E.g., `Foo {x: 1, y: 2}`, or `Foo {x: 1, .. rest}`.
-    Struct(Path, Vec<Field>, StructRest),
+    Struct(P<StructExpr>),
 
     /// An array literal constructed from one repeated element.
     ///
@@ -1709,13 +1715,6 @@ impl FloatTy {
             FloatTy::F64 => sym::f64,
         }
     }
-
-    pub fn bit_width(self) -> u64 {
-        match self {
-            FloatTy::F32 => 32,
-            FloatTy::F64 => 64,
-        }
-    }
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
@@ -1751,29 +1750,6 @@ impl IntTy {
             IntTy::I128 => sym::i128,
         }
     }
-
-    pub fn bit_width(&self) -> Option<u64> {
-        Some(match *self {
-            IntTy::Isize => return None,
-            IntTy::I8 => 8,
-            IntTy::I16 => 16,
-            IntTy::I32 => 32,
-            IntTy::I64 => 64,
-            IntTy::I128 => 128,
-        })
-    }
-
-    pub fn normalize(&self, target_width: u32) -> Self {
-        match self {
-            IntTy::Isize => match target_width {
-                16 => IntTy::I16,
-                32 => IntTy::I32,
-                64 => IntTy::I64,
-                _ => unreachable!(),
-            },
-            _ => *self,
-        }
-    }
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy, Debug)]
@@ -1809,29 +1785,6 @@ impl UintTy {
             UintTy::U128 => sym::u128,
         }
     }
-
-    pub fn bit_width(&self) -> Option<u64> {
-        Some(match *self {
-            UintTy::Usize => return None,
-            UintTy::U8 => 8,
-            UintTy::U16 => 16,
-            UintTy::U32 => 32,
-            UintTy::U64 => 64,
-            UintTy::U128 => 128,
-        })
-    }
-
-    pub fn normalize(&self, target_width: u32) -> Self {
-        match self {
-            UintTy::Usize => match target_width {
-                16 => UintTy::U16,
-                32 => UintTy::U32,
-                64 => UintTy::U64,
-                _ => unreachable!(),
-            },
-            _ => *self,
-        }
-    }
 }
 
 /// A constraint on an associated type (e.g., `A = Bar` in `Foo<A = Bar>` or
@@ -1951,7 +1904,7 @@ impl TyKind {
 }
 
 /// Syntax used to declare a trait object.
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
 pub enum TraitObjectSyntax {
     Dyn,
     None,
@@ -2046,7 +1999,7 @@ pub enum InlineAsmOperand {
         out_expr: Option<P<Expr>>,
     },
     Const {
-        expr: P<Expr>,
+        anon_const: AnonConst,
     },
     Sym {
         expr: P<Expr>,
@@ -2202,9 +2155,6 @@ pub struct FnDecl {
 }
 
 impl FnDecl {
-    pub fn get_self(&self) -> Option<ExplicitSelf> {
-        self.inputs.get(0).and_then(Param::to_self)
-    }
     pub fn has_self(&self) -> bool {
         self.inputs.get(0).map_or(false, Param::is_self)
     }
@@ -2299,7 +2249,7 @@ impl FnRetTy {
     }
 }
 
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
 pub enum Inline {
     Yes,
     No,
@@ -2521,11 +2471,11 @@ impl VisibilityKind {
     }
 }
 
-/// Field of a struct.
+/// Field definition in a struct, variant or union.
 ///
 /// E.g., `bar: usize` as in `struct Foo { bar: usize }`.
 #[derive(Clone, Encodable, Decodable, Debug)]
-pub struct StructField {
+pub struct FieldDef {
     pub attrs: Vec<Attribute>,
     pub id: NodeId,
     pub span: Span,
@@ -2542,11 +2492,11 @@ pub enum VariantData {
     /// Struct variant.
     ///
     /// E.g., `Bar { .. }` as in `enum Foo { Bar { .. } }`.
-    Struct(Vec<StructField>, bool),
+    Struct(Vec<FieldDef>, bool),
     /// Tuple variant.
     ///
     /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`.
-    Tuple(Vec<StructField>, NodeId),
+    Tuple(Vec<FieldDef>, NodeId),
     /// Unit variant.
     ///
     /// E.g., `Bar = ..` as in `enum Foo { Bar = .. }`.
@@ -2555,7 +2505,7 @@ pub enum VariantData {
 
 impl VariantData {
     /// Return the fields of this variant.
-    pub fn fields(&self) -> &[StructField] {
+    pub fn fields(&self) -> &[FieldDef] {
         match *self {
             VariantData::Struct(ref fields, ..) | VariantData::Tuple(ref fields, _) => fields,
             _ => &[],
@@ -2757,7 +2707,7 @@ pub enum ItemKind {
     MacroDef(MacroDef),
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(ItemKind, 112);
 
 impl ItemKind {
@@ -2831,7 +2781,7 @@ pub enum AssocItemKind {
     MacCall(MacCall),
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(AssocItemKind, 72);
 
 impl AssocItemKind {
@@ -2883,7 +2833,7 @@ pub enum ForeignItemKind {
     MacCall(MacCall),
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(ForeignItemKind, 72);
 
 impl From<ForeignItemKind> for ItemKind {
diff --git a/compiler/rustc_ast/src/ast_like.rs b/compiler/rustc_ast/src/ast_like.rs
index 6649cda69a0..945a44ab663 100644
--- a/compiler/rustc_ast/src/ast_like.rs
+++ b/compiler/rustc_ast/src/ast_like.rs
@@ -1,34 +1,89 @@
 use super::ptr::P;
+use super::token::Nonterminal;
 use super::tokenstream::LazyTokenStream;
-use super::{Arm, Field, FieldPat, GenericParam, Param, StructField, Variant};
-use super::{AssocItem, Expr, ForeignItem, Item, Local};
+use super::{Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
+use super::{AssocItem, Expr, ForeignItem, Item, Local, MacCallStmt};
 use super::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
 use super::{AttrVec, Attribute, Stmt, StmtKind};
 
+use std::fmt::Debug;
+
 /// An `AstLike` represents an AST node (or some wrapper around
 /// and AST node) which stores some combination of attributes
 /// and tokens.
-pub trait AstLike: Sized {
+pub trait AstLike: Sized + Debug {
+    /// This is `true` if this `AstLike` might support 'custom' (proc-macro) inner
+    /// attributes. Attributes like `#![cfg]` and `#![cfg_attr]` are not
+    /// considered 'custom' attributes
+    ///
+    /// If this is `false`, then this `AstLike` definitely does
+    /// not support 'custom' inner attributes, which enables some optimizations
+    /// during token collection.
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool;
     fn attrs(&self) -> &[Attribute];
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));
-    /// Called by `Parser::collect_tokens` to store the collected
-    /// tokens inside an AST node
-    fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {
-        // This default impl makes this trait easier to implement
-        // in tools like `rust-analyzer`
-        panic!("`finalize_tokens` is not supported!")
-    }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
 }
 
 impl<T: AstLike + 'static> AstLike for P<T> {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
     fn attrs(&self) -> &[Attribute] {
         (**self).attrs()
     }
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         (**self).visit_attrs(f);
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        (**self).finalize_tokens(tokens)
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        (**self).tokens_mut()
+    }
+}
+
+impl AstLike for crate::token::Nonterminal {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
+    fn attrs(&self) -> &[Attribute] {
+        match self {
+            Nonterminal::NtItem(item) => item.attrs(),
+            Nonterminal::NtStmt(stmt) => stmt.attrs(),
+            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.attrs(),
+            Nonterminal::NtPat(_)
+            | Nonterminal::NtTy(_)
+            | Nonterminal::NtMeta(_)
+            | Nonterminal::NtPath(_)
+            | Nonterminal::NtVis(_)
+            | Nonterminal::NtTT(_)
+            | Nonterminal::NtBlock(_)
+            | Nonterminal::NtIdent(..)
+            | Nonterminal::NtLifetime(_) => &[],
+        }
+    }
+    fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
+        match self {
+            Nonterminal::NtItem(item) => item.visit_attrs(f),
+            Nonterminal::NtStmt(stmt) => stmt.visit_attrs(f),
+            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.visit_attrs(f),
+            Nonterminal::NtPat(_)
+            | Nonterminal::NtTy(_)
+            | Nonterminal::NtMeta(_)
+            | Nonterminal::NtPath(_)
+            | Nonterminal::NtVis(_)
+            | Nonterminal::NtTT(_)
+            | Nonterminal::NtBlock(_)
+            | Nonterminal::NtIdent(..)
+            | Nonterminal::NtLifetime(_) => {}
+        }
+    }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        match self {
+            Nonterminal::NtItem(item) => item.tokens_mut(),
+            Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
+            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
+            Nonterminal::NtPat(pat) => pat.tokens_mut(),
+            Nonterminal::NtTy(ty) => ty.tokens_mut(),
+            Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
+            Nonterminal::NtPath(path) => path.tokens_mut(),
+            Nonterminal::NtVis(vis) => vis.tokens_mut(),
+            _ => panic!("Called tokens_mut on {:?}", self),
+        }
     }
 }
 
@@ -41,13 +96,17 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
 }
 
 impl AstLike for StmtKind {
+    // This might be an `StmtKind::Item`, which contains
+    // an item that supports inner attrs
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
+
     fn attrs(&self) -> &[Attribute] {
-        match *self {
-            StmtKind::Local(ref local) => local.attrs(),
-            StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.attrs(),
-            StmtKind::Item(ref item) => item.attrs(),
+        match self {
+            StmtKind::Local(local) => local.attrs(),
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.attrs(),
+            StmtKind::Item(item) => item.attrs(),
             StmtKind::Empty => &[],
-            StmtKind::MacCall(ref mac) => &*mac.attrs,
+            StmtKind::MacCall(mac) => &mac.attrs,
         }
     }
 
@@ -60,21 +119,20 @@ impl AstLike for StmtKind {
             StmtKind::MacCall(mac) => visit_attrvec(&mut mac.attrs, f),
         }
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        let stmt_tokens = match self {
-            StmtKind::Local(ref mut local) => &mut local.tokens,
-            StmtKind::Item(ref mut item) => &mut item.tokens,
-            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => &mut expr.tokens,
-            StmtKind::Empty => return,
-            StmtKind::MacCall(ref mut mac) => &mut mac.tokens,
-        };
-        if stmt_tokens.is_none() {
-            *stmt_tokens = Some(tokens);
-        }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        Some(match self {
+            StmtKind::Local(local) => &mut local.tokens,
+            StmtKind::Item(item) => &mut item.tokens,
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => &mut expr.tokens,
+            StmtKind::Empty => return None,
+            StmtKind::MacCall(mac) => &mut mac.tokens,
+        })
     }
 }
 
 impl AstLike for Stmt {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = StmtKind::SUPPORTS_CUSTOM_INNER_ATTRS;
+
     fn attrs(&self) -> &[Attribute] {
         self.kind.attrs()
     }
@@ -82,31 +140,31 @@ impl AstLike for Stmt {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         self.kind.visit_attrs(f);
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        self.kind.finalize_tokens(tokens)
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        self.kind.tokens_mut()
     }
 }
 
 impl AstLike for Attribute {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+
     fn attrs(&self) -> &[Attribute] {
         &[]
     }
     fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        match &mut self.kind {
-            AttrKind::Normal(_, attr_tokens) => {
-                if attr_tokens.is_none() {
-                    *attr_tokens = Some(tokens);
-                }
-            }
-            AttrKind::DocComment(..) => {
-                panic!("Called finalize_tokens on doc comment attr {:?}", self)
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        Some(match &mut self.kind {
+            AttrKind::Normal(_, tokens) => tokens,
+            kind @ AttrKind::DocComment(..) => {
+                panic!("Called tokens_mut on doc comment attr {:?}", kind)
             }
-        }
+        })
     }
 }
 
 impl<T: AstLike> AstLike for Option<T> {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
+
     fn attrs(&self) -> &[Attribute] {
         self.as_ref().map(|inner| inner.attrs()).unwrap_or(&[])
     }
@@ -115,10 +173,8 @@ impl<T: AstLike> AstLike for Option<T> {
             inner.visit_attrs(f);
         }
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        if let Some(inner) = self {
-            inner.finalize_tokens(tokens);
-        }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        self.as_mut().and_then(|inner| inner.tokens_mut())
     }
 }
 
@@ -142,8 +198,13 @@ impl VecOrAttrVec for AttrVec {
 }
 
 macro_rules! derive_has_tokens_and_attrs {
-    ($($ty:path),*) => { $(
+    (
+        const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs:literal;
+        $($ty:path),*
+    ) => { $(
         impl AstLike for $ty {
+            const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs;
+
             fn attrs(&self) -> &[Attribute] {
                 &self.attrs
             }
@@ -152,12 +213,10 @@ macro_rules! derive_has_tokens_and_attrs {
                 VecOrAttrVec::visit(&mut self.attrs, f)
             }
 
-            fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-                if self.tokens.is_none() {
-                    self.tokens = Some(tokens);
-                }
-
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                Some(&mut self.tokens)
             }
+
         }
     )* }
 }
@@ -165,6 +224,8 @@ macro_rules! derive_has_tokens_and_attrs {
 macro_rules! derive_has_attrs_no_tokens {
     ($($ty:path),*) => { $(
         impl AstLike for $ty {
+            const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+
             fn attrs(&self) -> &[Attribute] {
                 &self.attrs
             }
@@ -173,7 +234,9 @@ macro_rules! derive_has_attrs_no_tokens {
                 VecOrAttrVec::visit(&mut self.attrs, f)
             }
 
-            fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {}
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                None
+            }
         }
     )* }
 }
@@ -181,34 +244,38 @@ macro_rules! derive_has_attrs_no_tokens {
 macro_rules! derive_has_tokens_no_attrs {
     ($($ty:path),*) => { $(
         impl AstLike for $ty {
+            const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+
             fn attrs(&self) -> &[Attribute] {
                 &[]
             }
 
-            fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {
-            }
-
-            fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-                if self.tokens.is_none() {
-                    self.tokens = Some(tokens);
-                }
-
+            fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                Some(&mut self.tokens)
             }
         }
     )* }
 }
 
-// These AST nodes support both inert and active
-// attributes, so they also have tokens.
+// These ast nodes support both active and inert attributes,
+// so they have tokens collected to pass to proc macros
+derive_has_tokens_and_attrs! {
+    // Both `Item` and `AssocItem` can have bodies, which
+    // can contain inner attributes
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
+    Item, AssocItem, ForeignItem
+}
+
 derive_has_tokens_and_attrs! {
-    Item, Expr, Local, AssocItem, ForeignItem
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+    Local, MacCallStmt, Expr
 }
 
 // These ast nodes only support inert attributes, so they don't
 // store tokens (since nothing can observe them)
 derive_has_attrs_no_tokens! {
-    StructField, Arm,
-    Field, FieldPat, Variant, Param, GenericParam
+    FieldDef, Arm, ExprField, PatField, Variant, Param, GenericParam
 }
 
 // These AST nodes don't support attributes, but can
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index 1e224dbf833..41121d095f3 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -6,7 +6,9 @@ use crate::ast::{Lit, LitKind};
 use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
 use crate::ast::{Path, PathSegment};
 use crate::token::{self, CommentKind, Token};
-use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree, TreeAndSpacing};
+use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
+use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing};
+use crate::tokenstream::{LazyTokenStream, TokenStream};
 
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_span::source_map::BytePos;
@@ -33,10 +35,6 @@ impl MarkedAttrs {
     }
 }
 
-pub fn is_known_lint_tool(m_item: Ident) -> bool {
-    [sym::clippy, sym::rustc, sym::rustdoc].contains(&m_item.name)
-}
-
 impl NestedMetaItem {
     /// Returns the `MetaItem` if `self` is a `NestedMetaItem::MetaItem`.
     pub fn meta_item(&self) -> Option<&MetaItem> {
@@ -104,22 +102,14 @@ impl NestedMetaItem {
         self.meta_item().map_or(false, |meta_item| meta_item.is_word())
     }
 
-    /// Returns `true` if `self` is a `MetaItem` and the meta item is a `ValueString`.
-    pub fn is_value_str(&self) -> bool {
-        self.value_str().is_some()
-    }
-
-    /// Returns `true` if `self` is a `MetaItem` and the meta item is a list.
-    pub fn is_meta_item_list(&self) -> bool {
-        self.meta_item_list().is_some()
-    }
-
+    /// See [`MetaItem::name_value_literal_span`].
     pub fn name_value_literal_span(&self) -> Option<Span> {
         self.meta_item()?.name_value_literal_span()
     }
 }
 
 impl Attribute {
+    #[inline]
     pub fn has_name(&self, name: Symbol) -> bool {
         match self.kind {
             AttrKind::Normal(ref item, _) => item.path == name,
@@ -168,31 +158,6 @@ impl Attribute {
             false
         }
     }
-
-    pub fn is_meta_item_list(&self) -> bool {
-        self.meta_item_list().is_some()
-    }
-
-    /// Indicates if the attribute is a `ValueString`.
-    pub fn is_value_str(&self) -> bool {
-        self.value_str().is_some()
-    }
-
-    /// This is used in case you want the value span instead of the whole attribute. Example:
-    ///
-    /// ```text
-    /// #[doc(alias = "foo")]
-    /// ```
-    ///
-    /// In here, it'll return a span for `"foo"`.
-    pub fn name_value_literal_span(&self) -> Option<Span> {
-        match self.kind {
-            AttrKind::Normal(ref item, _) => {
-                item.meta(self.span).and_then(|meta| meta.name_value_literal_span())
-            }
-            AttrKind::DocComment(..) => None,
-        }
-    }
 }
 
 impl MetaItem {
@@ -239,10 +204,6 @@ impl MetaItem {
         self.path == name
     }
 
-    pub fn is_value_str(&self) -> bool {
-        self.value_str().is_some()
-    }
-
     /// This is used in case you want the value span instead of the whole attribute. Example:
     ///
     /// ```text
@@ -309,14 +270,18 @@ impl Attribute {
         }
     }
 
-    pub fn tokens(&self) -> TokenStream {
+    pub fn tokens(&self) -> AttrAnnotatedTokenStream {
         match self.kind {
             AttrKind::Normal(_, ref tokens) => tokens
                 .as_ref()
                 .unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
                 .create_token_stream(),
-            AttrKind::DocComment(comment_kind, data) => TokenStream::from(TokenTree::Token(
-                Token::new(token::DocComment(comment_kind, self.style, data), self.span),
+            AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from((
+                AttrAnnotatedTokenTree::Token(Token::new(
+                    token::DocComment(comment_kind, self.style, data),
+                    self.span,
+                )),
+                Spacing::Alone,
             )),
         }
     }
diff --git a/compiler/rustc_ast/src/expand/mod.rs b/compiler/rustc_ast/src/expand/mod.rs
index eebfc38bdf4..2ee1bfe0ae7 100644
--- a/compiler/rustc_ast/src/expand/mod.rs
+++ b/compiler/rustc_ast/src/expand/mod.rs
@@ -1,3 +1,3 @@
-//! Definitions shared by macros / syntax extensions and e.g. librustc_middle.
+//! Definitions shared by macros / syntax extensions and e.g. `rustc_middle`.
 
 pub mod allocator;
diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs
index 4eaef85043c..1e6da044ec0 100644
--- a/compiler/rustc_ast/src/lib.rs
+++ b/compiler/rustc_ast/src/lib.rs
@@ -14,9 +14,10 @@
 #![feature(const_fn_transmute)]
 #![feature(const_panic)]
 #![feature(crate_visibility_modifier)]
+#![feature(iter_zip)]
 #![feature(label_break_value)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 #[macro_use]
@@ -42,7 +43,6 @@ pub mod util {
 pub mod ast;
 pub mod ast_like;
 pub mod attr;
-pub mod crate_disambiguator;
 pub mod entry;
 pub mod expand;
 pub mod mut_visit;
@@ -59,7 +59,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
-/// instead of implementing everything in librustc_middle.
+/// instead of implementing everything in `rustc_middle`.
 pub trait HashStableContext: rustc_span::HashStableContext {
     fn hash_attr(&mut self, _: &ast::Attribute, hasher: &mut StableHasher);
 }
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index c286738811c..05f57f978c7 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -102,8 +102,8 @@ pub trait MutVisitor: Sized {
         noop_visit_fn_header(header, self);
     }
 
-    fn flat_map_struct_field(&mut self, sf: StructField) -> SmallVec<[StructField; 1]> {
-        noop_flat_map_struct_field(sf, self)
+    fn flat_map_field_def(&mut self, fd: FieldDef) -> SmallVec<[FieldDef; 1]> {
+        noop_flat_map_field_def(fd, self)
     }
 
     fn visit_item_kind(&mut self, i: &mut ItemKind) {
@@ -254,8 +254,8 @@ pub trait MutVisitor: Sized {
         noop_visit_mt(mt, self);
     }
 
-    fn flat_map_field(&mut self, f: Field) -> SmallVec<[Field; 1]> {
-        noop_flat_map_field(f, self)
+    fn flat_map_expr_field(&mut self, f: ExprField) -> SmallVec<[ExprField; 1]> {
+        noop_flat_map_expr_field(f, self)
     }
 
     fn visit_where_clause(&mut self, where_clause: &mut WhereClause) {
@@ -278,8 +278,8 @@ pub trait MutVisitor: Sized {
         // Do nothing.
     }
 
-    fn flat_map_field_pattern(&mut self, fp: FieldPat) -> SmallVec<[FieldPat; 1]> {
-        noop_flat_map_field_pattern(fp, self)
+    fn flat_map_pat_field(&mut self, fp: PatField) -> SmallVec<[PatField; 1]> {
+        noop_flat_map_pat_field(fp, self)
     }
 }
 
@@ -385,11 +385,11 @@ pub fn visit_delim_span<T: MutVisitor>(dspan: &mut DelimSpan, vis: &mut T) {
     vis.visit_span(&mut dspan.close);
 }
 
-pub fn noop_flat_map_field_pattern<T: MutVisitor>(
-    mut fp: FieldPat,
+pub fn noop_flat_map_pat_field<T: MutVisitor>(
+    mut fp: PatField,
     vis: &mut T,
-) -> SmallVec<[FieldPat; 1]> {
-    let FieldPat { attrs, id, ident, is_placeholder: _, is_shorthand: _, pat, span } = &mut fp;
+) -> SmallVec<[PatField; 1]> {
+    let PatField { attrs, id, ident, is_placeholder: _, is_shorthand: _, pat, span } = &mut fp;
     vis.visit_id(id);
     vis.visit_ident(ident);
     vis.visit_pat(pat);
@@ -631,6 +631,33 @@ pub fn noop_flat_map_param<T: MutVisitor>(mut param: Param, vis: &mut T) -> Smal
 }
 
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
+pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) {
+    match tt {
+        AttrAnnotatedTokenTree::Token(token) => {
+            visit_token(token, vis);
+        }
+        AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
+            vis.visit_span(open);
+            vis.visit_span(close);
+            visit_attr_annotated_tts(tts, vis);
+        }
+        AttrAnnotatedTokenTree::Attributes(data) => {
+            for attr in &mut *data.attrs {
+                match &mut attr.kind {
+                    AttrKind::Normal(_, attr_tokens) => {
+                        visit_lazy_tts(attr_tokens, vis);
+                    }
+                    AttrKind::DocComment(..) => {
+                        vis.visit_span(&mut attr.span);
+                    }
+                }
+            }
+            visit_lazy_tts_opt_mut(Some(&mut data.tokens), vis);
+        }
+    }
+}
+
+// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
     match tt {
         TokenTree::Token(token) => {
@@ -652,16 +679,30 @@ pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T)
     }
 }
 
-pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
+pub fn visit_attr_annotated_tts<T: MutVisitor>(
+    AttrAnnotatedTokenStream(tts): &mut AttrAnnotatedTokenStream,
+    vis: &mut T,
+) {
+    if vis.token_visiting_enabled() && !tts.is_empty() {
+        let tts = Lrc::make_mut(tts);
+        visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis));
+    }
+}
+
+pub fn visit_lazy_tts_opt_mut<T: MutVisitor>(lazy_tts: Option<&mut LazyTokenStream>, vis: &mut T) {
     if vis.token_visiting_enabled() {
-        visit_opt(lazy_tts, |lazy_tts| {
+        if let Some(lazy_tts) = lazy_tts {
             let mut tts = lazy_tts.create_token_stream();
-            visit_tts(&mut tts, vis);
+            visit_attr_annotated_tts(&mut tts, vis);
             *lazy_tts = LazyTokenStream::new(tts);
-        })
+        }
     }
 }
 
+pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
+    visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis);
+}
+
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 // Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
 // In practice the ident part is not actually used by specific visitors right now,
@@ -842,10 +883,10 @@ pub fn noop_visit_where_predicate<T: MutVisitor>(pred: &mut WherePredicate, vis:
 pub fn noop_visit_variant_data<T: MutVisitor>(vdata: &mut VariantData, vis: &mut T) {
     match vdata {
         VariantData::Struct(fields, ..) => {
-            fields.flat_map_in_place(|field| vis.flat_map_struct_field(field));
+            fields.flat_map_in_place(|field| vis.flat_map_field_def(field));
         }
         VariantData::Tuple(fields, id) => {
-            fields.flat_map_in_place(|field| vis.flat_map_struct_field(field));
+            fields.flat_map_in_place(|field| vis.flat_map_field_def(field));
             vis.visit_id(id);
         }
         VariantData::Unit(id) => vis.visit_id(id),
@@ -864,22 +905,25 @@ pub fn noop_visit_poly_trait_ref<T: MutVisitor>(p: &mut PolyTraitRef, vis: &mut
     vis.visit_span(span);
 }
 
-pub fn noop_flat_map_struct_field<T: MutVisitor>(
-    mut sf: StructField,
+pub fn noop_flat_map_field_def<T: MutVisitor>(
+    mut fd: FieldDef,
     visitor: &mut T,
-) -> SmallVec<[StructField; 1]> {
-    let StructField { span, ident, vis, id, ty, attrs, is_placeholder: _ } = &mut sf;
+) -> SmallVec<[FieldDef; 1]> {
+    let FieldDef { span, ident, vis, id, ty, attrs, is_placeholder: _ } = &mut fd;
     visitor.visit_span(span);
     visit_opt(ident, |ident| visitor.visit_ident(ident));
     visitor.visit_vis(vis);
     visitor.visit_id(id);
     visitor.visit_ty(ty);
     visit_attrs(attrs, visitor);
-    smallvec![sf]
+    smallvec![fd]
 }
 
-pub fn noop_flat_map_field<T: MutVisitor>(mut f: Field, vis: &mut T) -> SmallVec<[Field; 1]> {
-    let Field { ident, expr, span, is_shorthand: _, attrs, id, is_placeholder: _ } = &mut f;
+pub fn noop_flat_map_expr_field<T: MutVisitor>(
+    mut f: ExprField,
+    vis: &mut T,
+) -> SmallVec<[ExprField; 1]> {
+    let ExprField { ident, expr, span, is_shorthand: _, attrs, id, is_placeholder: _ } = &mut f;
     vis.visit_ident(ident);
     vis.visit_expr(expr);
     vis.visit_id(id);
@@ -1102,7 +1146,7 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
         }
         PatKind::Struct(path, fields, _etc) => {
             vis.visit_path(path);
-            fields.flat_map_in_place(|field| vis.flat_map_field_pattern(field));
+            fields.flat_map_in_place(|field| vis.flat_map_pat_field(field));
         }
         PatKind::Box(inner) => vis.visit_pat(inner),
         PatKind::Ref(inner, _mutbl) => vis.visit_pat(inner),
@@ -1249,7 +1293,6 @@ pub fn noop_visit_expr<T: MutVisitor>(
                 match op {
                     InlineAsmOperand::In { expr, .. }
                     | InlineAsmOperand::InOut { expr, .. }
-                    | InlineAsmOperand::Const { expr, .. }
                     | InlineAsmOperand::Sym { expr, .. } => vis.visit_expr(expr),
                     InlineAsmOperand::Out { expr, .. } => {
                         if let Some(expr) = expr {
@@ -1262,6 +1305,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
                             vis.visit_expr(out_expr);
                         }
                     }
+                    InlineAsmOperand::Const { anon_const, .. } => vis.visit_anon_const(anon_const),
                 }
             }
         }
@@ -1283,10 +1327,11 @@ pub fn noop_visit_expr<T: MutVisitor>(
             visit_vec(inputs, |(_c, expr)| vis.visit_expr(expr));
         }
         ExprKind::MacCall(mac) => vis.visit_mac_call(mac),
-        ExprKind::Struct(path, fields, expr) => {
+        ExprKind::Struct(se) => {
+            let StructExpr { path, fields, rest } = se.deref_mut();
             vis.visit_path(path);
-            fields.flat_map_in_place(|field| vis.flat_map_field(field));
-            match expr {
+            fields.flat_map_in_place(|field| vis.flat_map_expr_field(field));
+            match rest {
                 StructRest::Base(expr) => vis.visit_expr(expr),
                 StructRest::Rest(_span) => {}
                 StructRest::None => {}
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index 90bfb01d6c7..10d48a55bb5 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -11,11 +11,9 @@ use crate::tokenstream::TokenTree;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
 use rustc_macros::HashStable_Generic;
-use rustc_span::hygiene::ExpnKind;
-use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::symbol::{Ident, Symbol};
-use rustc_span::{self, edition::Edition, FileName, RealFileName, Span, DUMMY_SP};
+use rustc_span::{self, edition::Edition, Span, DUMMY_SP};
 use std::borrow::Cow;
 use std::{fmt, mem};
 
@@ -244,7 +242,7 @@ pub enum TokenKind {
 }
 
 // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(TokenKind, 16);
 
 #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@@ -682,7 +680,7 @@ pub enum Nonterminal {
 }
 
 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(Nonterminal, 48);
 
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
@@ -690,13 +688,13 @@ pub enum NonterminalKind {
     Item,
     Block,
     Stmt,
-    Pat2018 {
-        /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the
+    Pat2015 {
+        /// Keep track of whether the user used `:pat2015` or `:pat` and we inferred it from the
         /// edition of the span. This is used for diagnostics.
         inferred: bool,
     },
     Pat2021 {
-        /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the
+        /// Keep track of whether the user used `:pat2015` or `:pat` and we inferred it from the
         /// edition of the span. This is used for diagnostics.
         inferred: bool,
     },
@@ -724,11 +722,11 @@ impl NonterminalKind {
             sym::stmt => NonterminalKind::Stmt,
             sym::pat => match edition() {
                 Edition::Edition2015 | Edition::Edition2018 => {
-                    NonterminalKind::Pat2018 { inferred: true }
+                    NonterminalKind::Pat2015 { inferred: true }
                 }
                 Edition::Edition2021 => NonterminalKind::Pat2021 { inferred: true },
             },
-            sym::pat2018 => NonterminalKind::Pat2018 { inferred: false },
+            sym::pat2015 => NonterminalKind::Pat2015 { inferred: false },
             sym::pat2021 => NonterminalKind::Pat2021 { inferred: false },
             sym::expr => NonterminalKind::Expr,
             sym::ty => NonterminalKind::Ty,
@@ -747,9 +745,9 @@ impl NonterminalKind {
             NonterminalKind::Item => sym::item,
             NonterminalKind::Block => sym::block,
             NonterminalKind::Stmt => sym::stmt,
-            NonterminalKind::Pat2018 { inferred: false } => sym::pat2018,
+            NonterminalKind::Pat2015 { inferred: false } => sym::pat2015,
             NonterminalKind::Pat2021 { inferred: false } => sym::pat2021,
-            NonterminalKind::Pat2018 { inferred: true }
+            NonterminalKind::Pat2015 { inferred: true }
             | NonterminalKind::Pat2021 { inferred: true } => sym::pat,
             NonterminalKind::Expr => sym::expr,
             NonterminalKind::Ty => sym::ty,
@@ -786,79 +784,6 @@ impl Nonterminal {
             NtTT(tt) => tt.span(),
         }
     }
-
-    /// This nonterminal looks like some specific enums from
-    /// `proc-macro-hack` and `procedural-masquerade` crates.
-    /// We need to maintain some special pretty-printing behavior for them due to incorrect
-    /// asserts in old versions of those crates and their wide use in the ecosystem.
-    /// See issue #73345 for more details.
-    /// FIXME(#73933): Remove this eventually.
-    pub fn pretty_printing_compatibility_hack(&self) -> bool {
-        let item = match self {
-            NtItem(item) => item,
-            NtStmt(stmt) => match &stmt.kind {
-                ast::StmtKind::Item(item) => item,
-                _ => return false,
-            },
-            _ => return false,
-        };
-
-        let name = item.ident.name;
-        if name == sym::ProceduralMasqueradeDummyType || name == sym::ProcMacroHack {
-            if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
-                if let [variant] = &*enum_def.variants {
-                    return variant.ident.name == sym::Input;
-                }
-            }
-        }
-        false
-    }
-
-    // See issue #74616 for details
-    pub fn ident_name_compatibility_hack(
-        &self,
-        orig_span: Span,
-        source_map: &SourceMap,
-    ) -> Option<(Ident, bool)> {
-        if let NtIdent(ident, is_raw) = self {
-            if let ExpnKind::Macro(_, macro_name) = orig_span.ctxt().outer_expn_data().kind {
-                let filename = source_map.span_to_filename(orig_span);
-                if let FileName::Real(RealFileName::Named(path)) = filename {
-                    let matches_prefix = |prefix, filename| {
-                        // Check for a path that ends with 'prefix*/src/<filename>'
-                        let mut iter = path.components().rev();
-                        iter.next().and_then(|p| p.as_os_str().to_str()) == Some(filename)
-                            && iter.next().and_then(|p| p.as_os_str().to_str()) == Some("src")
-                            && iter
-                                .next()
-                                .and_then(|p| p.as_os_str().to_str())
-                                .map_or(false, |p| p.starts_with(prefix))
-                    };
-
-                    if (macro_name == sym::impl_macros
-                        && matches_prefix("time-macros-impl", "lib.rs"))
-                        || (macro_name == sym::arrays && matches_prefix("js-sys", "lib.rs"))
-                    {
-                        let snippet = source_map.span_to_snippet(orig_span);
-                        if snippet.as_deref() == Ok("$name") {
-                            return Some((*ident, *is_raw));
-                        }
-                    }
-
-                    if macro_name == sym::tuple_from_req
-                        && (matches_prefix("actix-web", "extract.rs")
-                            || matches_prefix("actori-web", "extract.rs"))
-                    {
-                        let snippet = source_map.span_to_snippet(orig_span);
-                        if snippet.as_deref() == Ok("$T") {
-                            return Some((*ident, *is_raw));
-                        }
-                    }
-                }
-            }
-        }
-        None
-    }
 }
 
 impl PartialEq for Nonterminal {
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 9ac05f316f0..2d463a4588c 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -14,6 +14,7 @@
 //! ownership of the original.
 
 use crate::token::{self, DelimToken, Token, TokenKind};
+use crate::AttrVec;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{self, Lrc};
@@ -89,10 +90,6 @@ impl TokenTree {
         }
     }
 
-    pub fn joint(self) -> TokenStream {
-        TokenStream::new(vec![(self, Spacing::Joint)])
-    }
-
     pub fn token(kind: TokenKind, span: Span) -> TokenTree {
         TokenTree::Token(Token::new(kind, span))
     }
@@ -127,11 +124,11 @@ where
 }
 
 pub trait CreateTokenStream: sync::Send + sync::Sync {
-    fn create_token_stream(&self) -> TokenStream;
+    fn create_token_stream(&self) -> AttrAnnotatedTokenStream;
 }
 
-impl CreateTokenStream for TokenStream {
-    fn create_token_stream(&self) -> TokenStream {
+impl CreateTokenStream for AttrAnnotatedTokenStream {
+    fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
         self.clone()
     }
 }
@@ -147,14 +144,14 @@ impl LazyTokenStream {
         LazyTokenStream(Lrc::new(Box::new(inner)))
     }
 
-    pub fn create_token_stream(&self) -> TokenStream {
+    pub fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
         self.0.create_token_stream()
     }
 }
 
 impl fmt::Debug for LazyTokenStream {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Debug::fmt("LazyTokenStream", f)
+        write!(f, "LazyTokenStream({:?})", self.create_token_stream())
     }
 }
 
@@ -177,19 +174,158 @@ impl<CTX> HashStable<CTX> for LazyTokenStream {
     }
 }
 
+/// A `AttrAnnotatedTokenStream` is similar to a `TokenStream`, but with extra
+/// information about the tokens for attribute targets. This is used
+/// during expansion to perform early cfg-expansion, and to process attributes
+/// during proc-macro invocations.
+#[derive(Clone, Debug, Default, Encodable, Decodable)]
+pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing)>>);
+
+/// Like `TokenTree`, but for `AttrAnnotatedTokenStream`
+#[derive(Clone, Debug, Encodable, Decodable)]
+pub enum AttrAnnotatedTokenTree {
+    Token(Token),
+    Delimited(DelimSpan, DelimToken, AttrAnnotatedTokenStream),
+    /// Stores the attributes for an attribute target,
+    /// along with the tokens for that attribute target.
+    /// See `AttributesData` for more information
+    Attributes(AttributesData),
+}
+
+impl AttrAnnotatedTokenStream {
+    pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream {
+        AttrAnnotatedTokenStream(Lrc::new(tokens))
+    }
+
+    /// Converts this `AttrAnnotatedTokenStream` to a plain `TokenStream
+    /// During conversion, `AttrAnnotatedTokenTree::Attributes` get 'flattened'
+    /// back to a `TokenStream` of the form `outer_attr attr_target`.
+    /// If there are inner attributes, they are inserted into the proper
+    /// place in the attribute target tokens.
+    pub fn to_tokenstream(&self) -> TokenStream {
+        let trees: Vec<_> = self
+            .0
+            .iter()
+            .flat_map(|tree| match &tree.0 {
+                AttrAnnotatedTokenTree::Token(inner) => {
+                    smallvec![(TokenTree::Token(inner.clone()), tree.1)].into_iter()
+                }
+                AttrAnnotatedTokenTree::Delimited(span, delim, stream) => smallvec![(
+                    TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),
+                    tree.1,
+                )]
+                .into_iter(),
+                AttrAnnotatedTokenTree::Attributes(data) => {
+                    let mut outer_attrs = Vec::new();
+                    let mut inner_attrs = Vec::new();
+                    let attrs: Vec<_> = data.attrs.clone().into();
+                    for attr in attrs {
+                        match attr.style {
+                            crate::AttrStyle::Outer => {
+                                assert!(
+                                    inner_attrs.len() == 0,
+                                    "Found outer attribute {:?} after inner attrs {:?}",
+                                    attr,
+                                    inner_attrs
+                                );
+                                outer_attrs.push(attr);
+                            }
+                            crate::AttrStyle::Inner => {
+                                inner_attrs.push(attr);
+                            }
+                        }
+                    }
+
+                    let mut target_tokens: Vec<_> = data
+                        .tokens
+                        .create_token_stream()
+                        .to_tokenstream()
+                        .0
+                        .iter()
+                        .cloned()
+                        .collect();
+                    if !inner_attrs.is_empty() {
+                        let mut found = false;
+                        // Check the last two trees (to account for a trailing semi)
+                        for (tree, _) in target_tokens.iter_mut().rev().take(2) {
+                            if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
+                                // Inner attributes are only supported on extern blocks, functions, impls,
+                                // and modules. All of these have their inner attributes placed at
+                                // the beginning of the rightmost outermost braced group:
+                                // e.g. fn foo() { #![my_attr} }
+                                //
+                                // Therefore, we can insert them back into the right location
+                                // without needing to do any extra position tracking.
+                                //
+                                // Note: Outline modules are an exception - they can
+                                // have attributes like `#![my_attr]` at the start of a file.
+                                // Support for custom attributes in this position is not
+                                // properly implemented - we always synthesize fake tokens,
+                                // so we never reach this code.
+
+                                let mut builder = TokenStreamBuilder::new();
+                                for inner_attr in &inner_attrs {
+                                    builder.push(inner_attr.tokens().to_tokenstream());
+                                }
+                                builder.push(delim_tokens.clone());
+                                *tree = TokenTree::Delimited(*span, *delim, builder.build());
+                                found = true;
+                                break;
+                            }
+                        }
+
+                        assert!(
+                            found,
+                            "Failed to find trailing delimited group in: {:?}",
+                            target_tokens
+                        );
+                    }
+                    let mut flat: SmallVec<[_; 1]> = SmallVec::new();
+                    for attr in outer_attrs {
+                        // FIXME: Make this more efficient
+                        flat.extend(attr.tokens().to_tokenstream().0.clone().iter().cloned());
+                    }
+                    flat.extend(target_tokens);
+                    flat.into_iter()
+                }
+            })
+            .collect();
+        TokenStream::new(trees)
+    }
+}
+
+/// Stores the tokens for an attribute target, along
+/// with its attributes.
+///
+/// This is constructed during parsing when we need to capture
+/// tokens.
+///
+/// For example, `#[cfg(FALSE)] struct Foo {}` would
+/// have an `attrs` field containing the `#[cfg(FALSE)]` attr,
+/// and a `tokens` field storing the (unparesd) tokens `struct Foo {}`
+#[derive(Clone, Debug, Encodable, Decodable)]
+pub struct AttributesData {
+    /// Attributes, both outer and inner.
+    /// These are stored in the original order that they were parsed in.
+    pub attrs: AttrVec,
+    /// The underlying tokens for the attribute target that `attrs`
+    /// are applied to
+    pub tokens: LazyTokenStream,
+}
+
 /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
 ///
 /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
 /// instead of a representation of the abstract syntax tree.
 /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
-/// backwards compatability.
+/// backwards compatibility.
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
 pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>);
 
 pub type TreeAndSpacing = (TokenTree, Spacing);
 
 // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(TokenStream, 8);
 
 #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)]
@@ -239,6 +375,12 @@ impl TokenStream {
     }
 }
 
+impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream {
+    fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream {
+        AttrAnnotatedTokenStream::new(vec![(tree, spacing)])
+    }
+}
+
 impl From<TokenTree> for TokenStream {
     fn from(tree: TokenTree) -> TokenStream {
         TokenStream::new(vec![(tree, Spacing::Alone)])
@@ -278,14 +420,6 @@ impl TokenStream {
         self.0.len()
     }
 
-    pub fn span(&self) -> Option<Span> {
-        match &**self.0 {
-            [] => None,
-            [(tt, _)] => Some(tt.span()),
-            [(tt_start, _), .., (tt_end, _)] => Some(tt_start.span().to(tt_end.span())),
-        }
-    }
-
     pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::default(),
@@ -325,10 +459,6 @@ impl TokenStream {
         }
     }
 
-    pub fn trees_ref(&self) -> CursorRef<'_> {
-        CursorRef::new(self)
-    }
-
     pub fn trees(&self) -> Cursor {
         self.clone().into_trees()
     }
@@ -341,7 +471,7 @@ impl TokenStream {
     pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
         let mut t1 = self.trees();
         let mut t2 = other.trees();
-        for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
+        for (t1, t2) in iter::zip(&mut t1, &mut t2) {
             if !t1.eq_unspanned(&t2) {
                 return false;
             }
@@ -427,10 +557,6 @@ pub struct CursorRef<'t> {
 }
 
 impl<'t> CursorRef<'t> {
-    fn new(stream: &TokenStream) -> CursorRef<'_> {
-        CursorRef { stream, index: 0 }
-    }
-
     fn next_with_spacing(&mut self) -> Option<&'t TreeAndSpacing> {
         self.stream.0.get(self.index).map(|tree| {
             self.index += 1;
@@ -477,6 +603,10 @@ impl Cursor {
         }
     }
 
+    pub fn index(&self) -> usize {
+        self.index
+    }
+
     pub fn append(&mut self, new_stream: TokenStream) {
         if new_stream.is_empty() {
             return;
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index 32b9dd46bae..3f35919ae6a 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -151,8 +151,8 @@ pub trait Visitor<'ast>: Sized {
     fn visit_variant_data(&mut self, s: &'ast VariantData) {
         walk_struct_def(self, s)
     }
-    fn visit_struct_field(&mut self, s: &'ast StructField) {
-        walk_struct_field(self, s)
+    fn visit_field_def(&mut self, s: &'ast FieldDef) {
+        walk_field_def(self, s)
     }
     fn visit_enum_def(
         &mut self,
@@ -208,11 +208,11 @@ pub trait Visitor<'ast>: Sized {
     fn visit_fn_header(&mut self, _header: &'ast FnHeader) {
         // Nothing to do
     }
-    fn visit_field(&mut self, f: &'ast Field) {
-        walk_field(self, f)
+    fn visit_expr_field(&mut self, f: &'ast ExprField) {
+        walk_expr_field(self, f)
     }
-    fn visit_field_pattern(&mut self, fp: &'ast FieldPat) {
-        walk_field_pattern(self, fp)
+    fn visit_pat_field(&mut self, fp: &'ast PatField) {
+        walk_pat_field(self, fp)
     }
 }
 
@@ -364,13 +364,13 @@ where
     walk_list!(visitor, visit_attribute, &variant.attrs);
 }
 
-pub fn walk_field<'a, V: Visitor<'a>>(visitor: &mut V, f: &'a Field) {
+pub fn walk_expr_field<'a, V: Visitor<'a>>(visitor: &mut V, f: &'a ExprField) {
     visitor.visit_expr(&f.expr);
     visitor.visit_ident(f.ident);
     walk_list!(visitor, visit_attribute, f.attrs.iter());
 }
 
-pub fn walk_field_pattern<'a, V: Visitor<'a>>(visitor: &mut V, fp: &'a FieldPat) {
+pub fn walk_pat_field<'a, V: Visitor<'a>>(visitor: &mut V, fp: &'a PatField) {
     visitor.visit_ident(fp.ident);
     visitor.visit_pat(&fp.pat);
     walk_list!(visitor, visit_attribute, fp.attrs.iter());
@@ -509,7 +509,7 @@ pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) {
         }
         PatKind::Struct(ref path, ref fields, _) => {
             visitor.visit_path(path, pattern.id);
-            walk_list!(visitor, visit_field_pattern, fields);
+            walk_list!(visitor, visit_pat_field, fields);
         }
         PatKind::Box(ref subpattern)
         | PatKind::Ref(ref subpattern, _)
@@ -668,16 +668,16 @@ pub fn walk_assoc_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a AssocItem,
 }
 
 pub fn walk_struct_def<'a, V: Visitor<'a>>(visitor: &mut V, struct_definition: &'a VariantData) {
-    walk_list!(visitor, visit_struct_field, struct_definition.fields());
+    walk_list!(visitor, visit_field_def, struct_definition.fields());
 }
 
-pub fn walk_struct_field<'a, V: Visitor<'a>>(visitor: &mut V, struct_field: &'a StructField) {
-    visitor.visit_vis(&struct_field.vis);
-    if let Some(ident) = struct_field.ident {
+pub fn walk_field_def<'a, V: Visitor<'a>>(visitor: &mut V, field: &'a FieldDef) {
+    visitor.visit_vis(&field.vis);
+    if let Some(ident) = field.ident {
         visitor.visit_ident(ident);
     }
-    visitor.visit_ty(&struct_field.ty);
-    walk_list!(visitor, visit_attribute, &struct_field.attrs);
+    visitor.visit_ty(&field.ty);
+    walk_list!(visitor, visit_attribute, &field.attrs);
 }
 
 pub fn walk_block<'a, V: Visitor<'a>>(visitor: &mut V, block: &'a Block) {
@@ -721,10 +721,10 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
             visitor.visit_expr(element);
             visitor.visit_anon_const(count)
         }
-        ExprKind::Struct(ref path, ref fields, ref optional_base) => {
-            visitor.visit_path(path, expression.id);
-            walk_list!(visitor, visit_field, fields);
-            match optional_base {
+        ExprKind::Struct(ref se) => {
+            visitor.visit_path(&se.path, expression.id);
+            walk_list!(visitor, visit_expr_field, &se.fields);
+            match &se.rest {
                 StructRest::Base(expr) => visitor.visit_expr(expr),
                 StructRest::Rest(_span) => {}
                 StructRest::None => {}
@@ -835,7 +835,6 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
                 match op {
                     InlineAsmOperand::In { expr, .. }
                     | InlineAsmOperand::InOut { expr, .. }
-                    | InlineAsmOperand::Const { expr, .. }
                     | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr),
                     InlineAsmOperand::Out { expr, .. } => {
                         if let Some(expr) = expr {
@@ -848,6 +847,9 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
                             visitor.visit_expr(out_expr);
                         }
                     }
+                    InlineAsmOperand::Const { anon_const, .. } => {
+                        visitor.visit_anon_const(anon_const)
+                    }
                 }
             }
         }
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 82b41e13ccc..75dfe951c94 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -97,6 +97,23 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     ExprKind::Let(ref pat, ref scrutinee) => {
                         self.lower_expr_if_let(e.span, pat, scrutinee, then, else_opt.as_deref())
                     }
+                    ExprKind::Paren(ref paren) => match paren.peel_parens().kind {
+                        ExprKind::Let(ref pat, ref scrutinee) => {
+                            // A user has written `if (let Some(x) = foo) {`, we want to avoid
+                            // confusing them with mentions of nightly features.
+                            // If this logic is changed, you will also likely need to touch
+                            // `unused::UnusedParens::check_expr`.
+                            self.if_let_expr_with_parens(cond, &paren.peel_parens());
+                            self.lower_expr_if_let(
+                                e.span,
+                                pat,
+                                scrutinee,
+                                then,
+                                else_opt.as_deref(),
+                            )
+                        }
+                        _ => self.lower_expr_if(cond, then, else_opt.as_deref()),
+                    },
                     _ => self.lower_expr_if(cond, then, else_opt.as_deref()),
                 },
                 ExprKind::While(ref cond, ref body, opt_label) => self
@@ -207,8 +224,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
                 ExprKind::InlineAsm(ref asm) => self.lower_expr_asm(e.span, asm),
                 ExprKind::LlvmInlineAsm(ref asm) => self.lower_expr_llvm_asm(asm),
-                ExprKind::Struct(ref path, ref fields, ref rest) => {
-                    let rest = match rest {
+                ExprKind::Struct(ref se) => {
+                    let rest = match &se.rest {
                         StructRest::Base(e) => Some(self.lower_expr(e)),
                         StructRest::Rest(sp) => {
                             self.sess
@@ -223,11 +240,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         self.arena.alloc(self.lower_qpath(
                             e.id,
                             &None,
-                            path,
+                            &se.path,
                             ParamMode::Optional,
                             ImplTraitContext::disallowed(),
                         )),
-                        self.arena.alloc_from_iter(fields.iter().map(|x| self.lower_field(x))),
+                        self.arena
+                            .alloc_from_iter(se.fields.iter().map(|x| self.lower_expr_field(x))),
                         rest,
                     )
                 }
@@ -241,9 +259,18 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         ex.span = e.span;
                     }
                     // Merge attributes into the inner expression.
-                    let mut attrs: Vec<_> = e.attrs.iter().map(|a| self.lower_attr(a)).collect();
-                    attrs.extend::<Vec<_>>(ex.attrs.into());
-                    ex.attrs = attrs.into();
+                    if !e.attrs.is_empty() {
+                        let old_attrs = self.attrs.get(&ex.hir_id).map(|la| *la).unwrap_or(&[]);
+                        self.attrs.insert(
+                            ex.hir_id,
+                            &*self.arena.alloc_from_iter(
+                                e.attrs
+                                    .iter()
+                                    .map(|a| self.lower_attr(a))
+                                    .chain(old_attrs.iter().cloned()),
+                            ),
+                        );
+                    }
                     return ex;
                 }
 
@@ -255,12 +282,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ExprKind::MacCall(_) => panic!("{:?} shouldn't exist here", e.span),
             };
 
-            hir::Expr {
-                hir_id: self.lower_node_id(e.id),
-                kind,
-                span: e.span,
-                attrs: e.attrs.iter().map(|a| self.lower_attr(a)).collect::<Vec<_>>().into(),
-            }
+            let hir_id = self.lower_node_id(e.id);
+            self.lower_attrs(hir_id, &e.attrs);
+            hir::Expr { hir_id, kind, span: e.span }
         })
     }
 
@@ -346,6 +370,25 @@ impl<'hir> LoweringContext<'_, 'hir> {
         hir::ExprKind::Call(f, self.lower_exprs(&real_args))
     }
 
+    fn if_let_expr_with_parens(&mut self, cond: &Expr, paren: &Expr) {
+        let start = cond.span.until(paren.span);
+        let end = paren.span.shrink_to_hi().until(cond.span.shrink_to_hi());
+        self.sess
+            .struct_span_err(
+                vec![start, end],
+                "invalid parentheses around `let` expression in `if let`",
+            )
+            .multipart_suggestion(
+                "`if let` needs to be written without parentheses",
+                vec![(start, String::new()), (end, String::new())],
+                rustc_errors::Applicability::MachineApplicable,
+            )
+            .emit();
+        // Ideally, we'd remove the feature gating of a `let` expression since we are already
+        // complaining about it here, but `feature_gate::check_crate` has already run by now:
+        // self.sess.parse_sess.gated_spans.ungate_last(sym::let_chains, paren.span);
+    }
+
     /// Emit an error and lower `ast::ExprKind::Let(pat, scrutinee)` into:
     /// ```rust
     /// match scrutinee { pats => true, _ => false }
@@ -356,8 +399,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
         if self.sess.opts.unstable_features.is_nightly_build() {
             self.sess
                 .struct_span_err(span, "`let` expressions are not supported here")
-                .note("only supported directly in conditions of `if`- and `while`-expressions")
-                .note("as well as when nested within `&&` and parenthesis in those conditions")
+                .note(
+                    "only supported directly without parentheses in conditions of `if`- and \
+                     `while`-expressions, as well as in `let` chains within parentheses",
+                )
                 .emit();
         } else {
             self.sess
@@ -580,14 +625,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 hir::Guard::If(self.lower_expr(cond))
             }
         });
-        hir::Arm {
-            hir_id: self.next_id(),
-            attrs: self.lower_attrs(&arm.attrs),
-            pat,
-            guard,
-            body: self.lower_expr(&arm.body),
-            span: arm.span,
-        }
+        let hir_id = self.next_id();
+        self.lower_attrs(hir_id, &arm.attrs);
+        hir::Arm { hir_id, pat, guard, body: self.lower_expr(&arm.body), span: arm.span }
     }
 
     /// Lower an `async` construct to a generator that is then wrapped so it implements `Future`.
@@ -631,7 +671,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             Ident::with_dummy_span(sym::_task_context),
             hir::BindingAnnotation::Mutable,
         );
-        let param = hir::Param { attrs: &[], hir_id: self.next_id(), pat, ty_span: span, span };
+        let param = hir::Param { hir_id: self.next_id(), pat, ty_span: span, span };
         let params = arena_vec![self; param];
 
         let body_id = self.lower_body(move |this| {
@@ -652,12 +692,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
             span,
             Some(hir::Movability::Static),
         );
-        let generator = hir::Expr {
-            hir_id: self.lower_node_id(closure_node_id),
-            kind: generator_kind,
-            span,
-            attrs: ThinVec::new(),
-        };
+        let generator =
+            hir::Expr { hir_id: self.lower_node_id(closure_node_id), kind: generator_kind, span };
 
         // `future::from_generator`:
         let unstable_span =
@@ -811,7 +847,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
             hir_id: loop_hir_id,
             kind: hir::ExprKind::Loop(loop_block, None, hir::LoopSource::Loop, span),
             span,
-            attrs: ThinVec::new(),
         });
 
         // mut pinned => loop { ... }
@@ -988,7 +1023,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         // Introduce a `let` for destructuring: `let (lhs1, lhs2) = t`.
         let destructure_let = self.stmt_let_pat(
-            ThinVec::new(),
+            None,
             whole_span,
             Some(rhs),
             pat,
@@ -1076,10 +1111,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
             }
             // Structs.
-            ExprKind::Struct(path, fields, rest) => {
-                let field_pats = self.arena.alloc_from_iter(fields.iter().map(|f| {
+            ExprKind::Struct(se) => {
+                let field_pats = self.arena.alloc_from_iter(se.fields.iter().map(|f| {
                     let pat = self.destructure_assign(&f.expr, eq_sign_span, assignments);
-                    hir::FieldPat {
+                    hir::PatField {
                         hir_id: self.next_id(),
                         ident: f.ident,
                         pat,
@@ -1090,11 +1125,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let qpath = self.lower_qpath(
                     lhs.id,
                     &None,
-                    path,
+                    &se.path,
                     ParamMode::Optional,
                     ImplTraitContext::disallowed(),
                 );
-                let fields_omitted = match rest {
+                let fields_omitted = match &se.rest {
                     StructRest::Base(e) => {
                         self.sess
                             .struct_span_err(
@@ -1210,7 +1245,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             e1.iter().map(|e| ("start", e)).chain(e2.iter().map(|e| ("end", e))).map(|(s, e)| {
                 let expr = self.lower_expr(&e);
                 let ident = Ident::new(Symbol::intern(s), e.span);
-                self.field(ident, expr, e.span)
+                self.expr_field(ident, expr, e.span)
             }),
         );
 
@@ -1297,104 +1332,97 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_expr_asm(&mut self, sp: Span, asm: &InlineAsm) -> hir::ExprKind<'hir> {
-        if self.sess.asm_arch.is_none() {
+        // Rustdoc needs to support asm! from foriegn architectures: don't try
+        // lowering the register contraints in this case.
+        let asm_arch = if self.sess.opts.actually_rustdoc { None } else { self.sess.asm_arch };
+        if asm_arch.is_none() && !self.sess.opts.actually_rustdoc {
             struct_span_err!(self.sess, sp, E0472, "asm! is unsupported on this target").emit();
         }
         if asm.options.contains(InlineAsmOptions::ATT_SYNTAX)
-            && !matches!(
-                self.sess.asm_arch,
-                Some(asm::InlineAsmArch::X86 | asm::InlineAsmArch::X86_64)
-            )
+            && !matches!(asm_arch, Some(asm::InlineAsmArch::X86 | asm::InlineAsmArch::X86_64))
+            && !self.sess.opts.actually_rustdoc
         {
             self.sess
                 .struct_span_err(sp, "the `att_syntax` option is only supported on x86")
                 .emit();
         }
 
-        // Lower operands to HIR, filter_map skips any operands with invalid
-        // register classes.
+        // Lower operands to HIR. We use dummy register classes if an error
+        // occurs during lowering because we still need to be able to produce a
+        // valid HIR.
         let sess = self.sess;
         let operands: Vec<_> = asm
             .operands
             .iter()
-            .filter_map(|(op, op_sp)| {
-                let lower_reg = |reg| {
-                    Some(match reg {
-                        InlineAsmRegOrRegClass::Reg(s) => asm::InlineAsmRegOrRegClass::Reg(
+            .map(|(op, op_sp)| {
+                let lower_reg = |reg| match reg {
+                    InlineAsmRegOrRegClass::Reg(s) => {
+                        asm::InlineAsmRegOrRegClass::Reg(if let Some(asm_arch) = asm_arch {
                             asm::InlineAsmReg::parse(
-                                sess.asm_arch?,
+                                asm_arch,
                                 |feature| sess.target_features.contains(&Symbol::intern(feature)),
                                 &sess.target,
                                 s,
                             )
-                            .map_err(|e| {
+                            .unwrap_or_else(|e| {
                                 let msg = format!("invalid register `{}`: {}", s.as_str(), e);
                                 sess.struct_span_err(*op_sp, &msg).emit();
+                                asm::InlineAsmReg::Err
                             })
-                            .ok()?,
-                        ),
-                        InlineAsmRegOrRegClass::RegClass(s) => {
-                            asm::InlineAsmRegOrRegClass::RegClass(
-                                asm::InlineAsmRegClass::parse(sess.asm_arch?, s)
-                                    .map_err(|e| {
-                                        let msg = format!(
-                                            "invalid register class `{}`: {}",
-                                            s.as_str(),
-                                            e
-                                        );
-                                        sess.struct_span_err(*op_sp, &msg).emit();
-                                    })
-                                    .ok()?,
-                            )
-                        }
-                    })
+                        } else {
+                            asm::InlineAsmReg::Err
+                        })
+                    }
+                    InlineAsmRegOrRegClass::RegClass(s) => {
+                        asm::InlineAsmRegOrRegClass::RegClass(if let Some(asm_arch) = asm_arch {
+                            asm::InlineAsmRegClass::parse(asm_arch, s).unwrap_or_else(|e| {
+                                let msg = format!("invalid register class `{}`: {}", s.as_str(), e);
+                                sess.struct_span_err(*op_sp, &msg).emit();
+                                asm::InlineAsmRegClass::Err
+                            })
+                        } else {
+                            asm::InlineAsmRegClass::Err
+                        })
+                    }
                 };
 
-                // lower_reg is executed last because we need to lower all
-                // sub-expressions even if we throw them away later.
                 let op = match *op {
                     InlineAsmOperand::In { reg, ref expr } => hir::InlineAsmOperand::In {
+                        reg: lower_reg(reg),
                         expr: self.lower_expr_mut(expr),
-                        reg: lower_reg(reg)?,
                     },
                     InlineAsmOperand::Out { reg, late, ref expr } => hir::InlineAsmOperand::Out {
+                        reg: lower_reg(reg),
                         late,
                         expr: expr.as_ref().map(|expr| self.lower_expr_mut(expr)),
-                        reg: lower_reg(reg)?,
                     },
                     InlineAsmOperand::InOut { reg, late, ref expr } => {
                         hir::InlineAsmOperand::InOut {
+                            reg: lower_reg(reg),
                             late,
                             expr: self.lower_expr_mut(expr),
-                            reg: lower_reg(reg)?,
                         }
                     }
                     InlineAsmOperand::SplitInOut { reg, late, ref in_expr, ref out_expr } => {
                         hir::InlineAsmOperand::SplitInOut {
+                            reg: lower_reg(reg),
                             late,
                             in_expr: self.lower_expr_mut(in_expr),
                             out_expr: out_expr.as_ref().map(|expr| self.lower_expr_mut(expr)),
-                            reg: lower_reg(reg)?,
                         }
                     }
-                    InlineAsmOperand::Const { ref expr } => {
-                        hir::InlineAsmOperand::Const { expr: self.lower_expr_mut(expr) }
-                    }
+                    InlineAsmOperand::Const { ref anon_const } => hir::InlineAsmOperand::Const {
+                        anon_const: self.lower_anon_const(anon_const),
+                    },
                     InlineAsmOperand::Sym { ref expr } => {
                         hir::InlineAsmOperand::Sym { expr: self.lower_expr_mut(expr) }
                     }
                 };
-                Some((op, *op_sp))
+                (op, *op_sp)
             })
             .collect();
 
-        // Stop if there were any errors when lowering the register classes
-        if operands.len() != asm.operands.len() || sess.asm_arch.is_none() {
-            return hir::ExprKind::Err;
-        }
-
         // Validate template modifiers against the register classes for the operands
-        let asm_arch = sess.asm_arch.unwrap();
         for p in &asm.template {
             if let InlineAsmTemplatePiece::Placeholder {
                 operand_idx,
@@ -1409,7 +1437,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     | hir::InlineAsmOperand::InOut { reg, .. }
                     | hir::InlineAsmOperand::SplitInOut { reg, .. } => {
                         let class = reg.reg_class();
-                        let valid_modifiers = class.valid_modifiers(asm_arch);
+                        if class == asm::InlineAsmRegClass::Err {
+                            continue;
+                        }
+                        let valid_modifiers = class.valid_modifiers(asm_arch.unwrap());
                         if !valid_modifiers.contains(&modifier) {
                             let mut err = sess.struct_span_err(
                                 placeholder_span,
@@ -1468,43 +1499,64 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // previous iteration.
                 required_features.clear();
 
-                // Validate register classes against currently enabled target
-                // features. We check that at least one type is available for
-                // the current target.
                 let reg_class = reg.reg_class();
-                for &(_, feature) in reg_class.supported_types(asm_arch) {
-                    if let Some(feature) = feature {
-                        if self.sess.target_features.contains(&Symbol::intern(feature)) {
+                if reg_class == asm::InlineAsmRegClass::Err {
+                    continue;
+                }
+
+                // We ignore target feature requirements for clobbers: if the
+                // feature is disabled then the compiler doesn't care what we
+                // do with the registers.
+                //
+                // Note that this is only possible for explicit register
+                // operands, which cannot be used in the asm string.
+                let is_clobber = matches!(
+                    op,
+                    hir::InlineAsmOperand::Out {
+                        reg: asm::InlineAsmRegOrRegClass::Reg(_),
+                        late: _,
+                        expr: None
+                    }
+                );
+
+                if !is_clobber {
+                    // Validate register classes against currently enabled target
+                    // features. We check that at least one type is available for
+                    // the current target.
+                    for &(_, feature) in reg_class.supported_types(asm_arch.unwrap()) {
+                        if let Some(feature) = feature {
+                            if self.sess.target_features.contains(&Symbol::intern(feature)) {
+                                required_features.clear();
+                                break;
+                            } else {
+                                required_features.push(feature);
+                            }
+                        } else {
                             required_features.clear();
                             break;
-                        } else {
-                            required_features.push(feature);
                         }
-                    } else {
-                        required_features.clear();
-                        break;
-                    }
-                }
-                // We are sorting primitive strs here and can use unstable sort here
-                required_features.sort_unstable();
-                required_features.dedup();
-                match &required_features[..] {
-                    [] => {}
-                    [feature] => {
-                        let msg = format!(
-                            "register class `{}` requires the `{}` target feature",
-                            reg_class.name(),
-                            feature
-                        );
-                        sess.struct_span_err(op_sp, &msg).emit();
                     }
-                    features => {
-                        let msg = format!(
-                            "register class `{}` requires at least one target feature: {}",
-                            reg_class.name(),
-                            features.join(", ")
-                        );
-                        sess.struct_span_err(op_sp, &msg).emit();
+                    // We are sorting primitive strs here and can use unstable sort here
+                    required_features.sort_unstable();
+                    required_features.dedup();
+                    match &required_features[..] {
+                        [] => {}
+                        [feature] => {
+                            let msg = format!(
+                                "register class `{}` requires the `{}` target feature",
+                                reg_class.name(),
+                                feature
+                            );
+                            sess.struct_span_err(op_sp, &msg).emit();
+                        }
+                        features => {
+                            let msg = format!(
+                                "register class `{}` requires at least one target feature: {}",
+                                reg_class.name(),
+                                features.join(", ")
+                            );
+                            sess.struct_span_err(op_sp, &msg).emit();
+                        }
                     }
                 }
 
@@ -1624,8 +1676,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         hir::ExprKind::LlvmInlineAsm(self.arena.alloc(hir_asm))
     }
 
-    fn lower_field(&mut self, f: &Field) -> hir::Field<'hir> {
-        hir::Field {
+    fn lower_expr_field(&mut self, f: &ExprField) -> hir::ExprField<'hir> {
+        hir::ExprField {
             hir_id: self.next_id(),
             ident: f.ident,
             expr: self.lower_expr(&f.expr),
@@ -1747,7 +1799,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         // `let mut __next`
         let next_let = self.stmt_let_pat(
-            ThinVec::new(),
+            None,
             desugared_span,
             None,
             next_pat,
@@ -1757,7 +1809,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // `let <pat> = __next`
         let pat = self.lower_pat(pat);
         let pat_let = self.stmt_let_pat(
-            ThinVec::new(),
+            None,
             desugared_span,
             Some(next_expr),
             pat,
@@ -1781,12 +1833,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
             hir::LoopSource::ForLoop,
             e.span.with_hi(orig_head_span.hi()),
         );
-        let loop_expr = self.arena.alloc(hir::Expr {
-            hir_id: self.lower_node_id(e.id),
-            kind,
-            span: e.span,
-            attrs: ThinVec::new(),
-        });
+        let loop_expr =
+            self.arena.alloc(hir::Expr { hir_id: self.lower_node_id(e.id), kind, span: e.span });
 
         // `mut iter => { ... }`
         let iter_arm = self.arm(iter_pat, loop_expr);
@@ -2121,21 +2169,21 @@ impl<'hir> LoweringContext<'_, 'hir> {
         kind: hir::ExprKind<'hir>,
         attrs: AttrVec,
     ) -> hir::Expr<'hir> {
-        hir::Expr { hir_id: self.next_id(), kind, span, attrs }
+        let hir_id = self.next_id();
+        self.lower_attrs(hir_id, &attrs);
+        hir::Expr { hir_id, kind, span }
     }
 
-    fn field(&mut self, ident: Ident, expr: &'hir hir::Expr<'hir>, span: Span) -> hir::Field<'hir> {
-        hir::Field { hir_id: self.next_id(), ident, span, expr, is_shorthand: false }
+    fn expr_field(
+        &mut self,
+        ident: Ident,
+        expr: &'hir hir::Expr<'hir>,
+        span: Span,
+    ) -> hir::ExprField<'hir> {
+        hir::ExprField { hir_id: self.next_id(), ident, span, expr, is_shorthand: false }
     }
 
     fn arm(&mut self, pat: &'hir hir::Pat<'hir>, expr: &'hir hir::Expr<'hir>) -> hir::Arm<'hir> {
-        hir::Arm {
-            hir_id: self.next_id(),
-            attrs: &[],
-            pat,
-            guard: None,
-            span: expr.span,
-            body: expr,
-        }
+        hir::Arm { hir_id: self.next_id(), pat, guard: None, span: expr.span, body: expr }
     }
 }
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 8b740b77740..5fd8f7eb33a 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -18,6 +18,7 @@ use rustc_target::spec::abi;
 use smallvec::{smallvec, SmallVec};
 use tracing::debug;
 
+use std::iter;
 use std::mem;
 
 pub(super) struct ItemLowerer<'a, 'lowering, 'hir> {
@@ -206,7 +207,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             UseTreeKind::Glob => {}
             UseTreeKind::Simple(_, id1, id2) => {
                 for (_, &id) in
-                    self.expect_full_res_from_use(base_id).skip(1).zip([id1, id2].iter())
+                    iter::zip(self.expect_full_res_from_use(base_id).skip(1), &[id1, id2])
                 {
                     vec.push(id);
                 }
@@ -217,44 +218,41 @@ impl<'hir> LoweringContext<'_, 'hir> {
     pub fn lower_item(&mut self, i: &Item) -> Option<hir::Item<'hir>> {
         let mut ident = i.ident;
         let mut vis = self.lower_visibility(&i.vis, None);
-        let attrs = self.lower_attrs(&i.attrs);
 
         if let ItemKind::MacroDef(MacroDef { ref body, macro_rules }) = i.kind {
             if !macro_rules || self.sess.contains_name(&i.attrs, sym::macro_export) {
-                let def_id = self.lower_node_id(i.id).expect_owner();
+                let hir_id = self.lower_node_id(i.id);
+                self.lower_attrs(hir_id, &i.attrs);
                 let body = P(self.lower_mac_args(body));
                 self.exported_macros.push(hir::MacroDef {
                     ident,
                     vis,
-                    attrs,
-                    def_id,
+                    def_id: hir_id.expect_owner(),
                     span: i.span,
                     ast: MacroDef { body, macro_rules },
                 });
             } else {
-                self.non_exported_macro_attrs.extend(attrs.iter().cloned());
+                for a in i.attrs.iter() {
+                    let a = self.lower_attr(a);
+                    self.non_exported_macro_attrs.push(a);
+                }
             }
             return None;
         }
 
-        let kind = self.lower_item_kind(i.span, i.id, &mut ident, attrs, &mut vis, &i.kind);
-
-        Some(hir::Item {
-            def_id: self.lower_node_id(i.id).expect_owner(),
-            ident,
-            attrs,
-            kind,
-            vis,
-            span: i.span,
-        })
+        let hir_id = self.lower_node_id(i.id);
+        let attrs = self.lower_attrs(hir_id, &i.attrs);
+        let kind = self.lower_item_kind(i.span, i.id, hir_id, &mut ident, attrs, &mut vis, &i.kind);
+        Some(hir::Item { def_id: hir_id.expect_owner(), ident, kind, vis, span: i.span })
     }
 
     fn lower_item_kind(
         &mut self,
         span: Span,
         id: NodeId,
+        hir_id: hir::HirId,
         ident: &mut Ident,
-        attrs: &'hir [Attribute],
+        attrs: Option<&'hir [Attribute]>,
         vis: &mut hir::Visibility<'hir>,
         i: &ItemKind,
     ) -> hir::ItemKind<'hir> {
@@ -322,10 +320,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
             },
             ItemKind::ForeignMod(ref fm) => {
                 if fm.abi.is_none() {
-                    self.maybe_lint_missing_abi(span, id, abi::Abi::C);
+                    self.maybe_lint_missing_abi(span, id, abi::Abi::C { unwind: false });
                 }
                 hir::ItemKind::ForeignMod {
-                    abi: fm.abi.map_or(abi::Abi::C, |abi| self.lower_abi(abi)),
+                    abi: fm.abi.map_or(abi::Abi::C { unwind: false }, |abi| self.lower_abi(abi)),
                     items: self
                         .arena
                         .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))),
@@ -345,7 +343,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     ty,
                     ImplTraitContext::OtherOpaqueTy {
                         capturable_lifetimes: &mut FxHashSet::default(),
-                        origin: hir::OpaqueTyOrigin::Misc,
+                        origin: hir::OpaqueTyOrigin::TyAlias,
                     },
                 );
                 let generics = self.lower_generics(gen, ImplTraitContext::disallowed());
@@ -365,14 +363,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 self.lower_generics(generics, ImplTraitContext::disallowed()),
             ),
             ItemKind::Struct(ref struct_def, ref generics) => {
-                let struct_def = self.lower_variant_data(struct_def);
+                let struct_def = self.lower_variant_data(hir_id, struct_def);
                 hir::ItemKind::Struct(
                     struct_def,
                     self.lower_generics(generics, ImplTraitContext::disallowed()),
                 )
             }
             ItemKind::Union(ref vdata, ref generics) => {
-                let vdata = self.lower_variant_data(vdata);
+                let vdata = self.lower_variant_data(hir_id, vdata);
                 hir::ItemKind::Union(
                     vdata,
                     self.lower_generics(generics, ImplTraitContext::disallowed()),
@@ -505,7 +503,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         id: NodeId,
         vis: &mut hir::Visibility<'hir>,
         ident: &mut Ident,
-        attrs: &'hir [Attribute],
+        attrs: Option<&'hir [Attribute]>,
     ) -> hir::ItemKind<'hir> {
         debug!("lower_use_tree(tree={:?})", tree);
         debug!("lower_use_tree: vis = {:?}", vis);
@@ -540,7 +538,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // won't be dealing with macros in the rest of the compiler.
                 // Essentially a single `use` which imports two names is desugared into
                 // two imports.
-                for (res, &new_node_id) in resolutions.zip([id1, id2].iter()) {
+                for (res, &new_node_id) in iter::zip(resolutions, &[id1, id2]) {
                     let ident = *ident;
                     let mut path = path.clone();
                     for seg in &mut path.segments {
@@ -554,11 +552,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         let path = this.lower_path_extra(res, &path, ParamMode::Explicit, None);
                         let kind = hir::ItemKind::Use(path, hir::UseKind::Single);
                         let vis = this.rebuild_vis(&vis);
+                        if let Some(attrs) = attrs {
+                            this.attrs.insert(new_id, attrs);
+                        }
 
                         this.insert_item(hir::Item {
                             def_id: new_id.expect_owner(),
                             ident,
-                            attrs,
                             kind,
                             vis,
                             span,
@@ -626,11 +626,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
                         let kind =
                             this.lower_use_tree(use_tree, &prefix, id, &mut vis, &mut ident, attrs);
+                        if let Some(attrs) = attrs {
+                            this.attrs.insert(new_hir_id, attrs);
+                        }
 
                         this.insert_item(hir::Item {
                             def_id: new_hir_id.expect_owner(),
                             ident,
-                            attrs,
                             kind,
                             vis,
                             span: use_tree.span,
@@ -699,11 +701,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_foreign_item(&mut self, i: &ForeignItem) -> hir::ForeignItem<'hir> {
-        let def_id = self.resolver.local_def_id(i.id);
+        let hir_id = self.lower_node_id(i.id);
+        let def_id = hir_id.expect_owner();
+        self.lower_attrs(hir_id, &i.attrs);
         hir::ForeignItem {
             def_id,
             ident: i.ident,
-            attrs: self.lower_attrs(&i.attrs),
             kind: match i.kind {
                 ForeignItemKind::Fn(box FnKind(_, ref sig, ref generics, _)) => {
                     let fdec = &sig.decl;
@@ -748,33 +751,47 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_variant(&mut self, v: &Variant) -> hir::Variant<'hir> {
+        let id = self.lower_node_id(v.id);
+        self.lower_attrs(id, &v.attrs);
         hir::Variant {
-            attrs: self.lower_attrs(&v.attrs),
-            data: self.lower_variant_data(&v.data),
+            id,
+            data: self.lower_variant_data(id, &v.data),
             disr_expr: v.disr_expr.as_ref().map(|e| self.lower_anon_const(e)),
-            id: self.lower_node_id(v.id),
             ident: v.ident,
             span: v.span,
         }
     }
 
-    fn lower_variant_data(&mut self, vdata: &VariantData) -> hir::VariantData<'hir> {
+    fn lower_variant_data(
+        &mut self,
+        parent_id: hir::HirId,
+        vdata: &VariantData,
+    ) -> hir::VariantData<'hir> {
         match *vdata {
             VariantData::Struct(ref fields, recovered) => hir::VariantData::Struct(
                 self.arena
-                    .alloc_from_iter(fields.iter().enumerate().map(|f| self.lower_struct_field(f))),
+                    .alloc_from_iter(fields.iter().enumerate().map(|f| self.lower_field_def(f))),
                 recovered,
             ),
-            VariantData::Tuple(ref fields, id) => hir::VariantData::Tuple(
-                self.arena
-                    .alloc_from_iter(fields.iter().enumerate().map(|f| self.lower_struct_field(f))),
-                self.lower_node_id(id),
-            ),
-            VariantData::Unit(id) => hir::VariantData::Unit(self.lower_node_id(id)),
+            VariantData::Tuple(ref fields, id) => {
+                let ctor_id = self.lower_node_id(id);
+                self.alias_attrs(ctor_id, parent_id);
+                hir::VariantData::Tuple(
+                    self.arena.alloc_from_iter(
+                        fields.iter().enumerate().map(|f| self.lower_field_def(f)),
+                    ),
+                    ctor_id,
+                )
+            }
+            VariantData::Unit(id) => {
+                let ctor_id = self.lower_node_id(id);
+                self.alias_attrs(ctor_id, parent_id);
+                hir::VariantData::Unit(ctor_id)
+            }
         }
     }
 
-    fn lower_struct_field(&mut self, (index, f): (usize, &StructField)) -> hir::StructField<'hir> {
+    fn lower_field_def(&mut self, (index, f): (usize, &FieldDef)) -> hir::FieldDef<'hir> {
         let ty = if let TyKind::Path(ref qself, ref path) = f.ty.kind {
             let t = self.lower_path_ty(
                 &f.ty,
@@ -787,9 +804,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         } else {
             self.lower_ty(&f.ty, ImplTraitContext::disallowed())
         };
-        hir::StructField {
+        let hir_id = self.lower_node_id(f.id);
+        self.lower_attrs(hir_id, &f.attrs);
+        hir::FieldDef {
             span: f.span,
-            hir_id: self.lower_node_id(f.id),
+            hir_id,
             ident: match f.ident {
                 Some(ident) => ident,
                 // FIXME(jseyfried): positional field hygiene.
@@ -797,12 +816,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
             },
             vis: self.lower_visibility(&f.vis, None),
             ty,
-            attrs: self.lower_attrs(&f.attrs),
         }
     }
 
     fn lower_trait_item(&mut self, i: &AssocItem) -> hir::TraitItem<'hir> {
-        let trait_item_def_id = self.resolver.local_def_id(i.id);
+        let hir_id = self.lower_node_id(i.id);
+        let trait_item_def_id = hir_id.expect_owner();
 
         let (generics, kind) = match i.kind {
             AssocItemKind::Const(_, ref ty, ref default) => {
@@ -817,9 +836,17 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Required(names)))
             }
             AssocItemKind::Fn(box FnKind(_, ref sig, ref generics, Some(ref body))) => {
-                let body_id = self.lower_fn_body_block(i.span, &sig.decl, Some(body));
-                let (generics, sig) =
-                    self.lower_method_sig(generics, sig, trait_item_def_id, false, None, i.id);
+                let asyncness = sig.header.asyncness;
+                let body_id =
+                    self.lower_maybe_async_body(i.span, &sig.decl, asyncness, Some(&body));
+                let (generics, sig) = self.lower_method_sig(
+                    generics,
+                    sig,
+                    trait_item_def_id,
+                    false,
+                    asyncness.opt_return_id(),
+                    i.id,
+                );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)))
             }
             AssocItemKind::TyAlias(box TyAliasKind(_, ref generics, ref bounds, ref default)) => {
@@ -835,14 +862,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
             AssocItemKind::MacCall(..) => panic!("macro item shouldn't exist at this point"),
         };
 
-        hir::TraitItem {
-            def_id: trait_item_def_id,
-            ident: i.ident,
-            attrs: self.lower_attrs(&i.attrs),
-            generics,
-            kind,
-            span: i.span,
-        }
+        self.lower_attrs(hir_id, &i.attrs);
+        hir::TraitItem { def_id: trait_item_def_id, ident: i.ident, generics, kind, span: i.span }
     }
 
     fn lower_trait_item_ref(&mut self, i: &AssocItem) -> hir::TraitItemRef {
@@ -906,7 +927,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             ty,
                             ImplTraitContext::OtherOpaqueTy {
                                 capturable_lifetimes: &mut FxHashSet::default(),
-                                origin: hir::OpaqueTyOrigin::Misc,
+                                origin: hir::OpaqueTyOrigin::TyAlias,
                             },
                         );
                         hir::ImplItemKind::TyAlias(ty)
@@ -920,10 +941,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // Since `default impl` is not yet implemented, this is always true in impls.
         let has_value = true;
         let (defaultness, _) = self.lower_defaultness(i.kind.defaultness(), has_value);
+        let hir_id = self.lower_node_id(i.id);
+        self.lower_attrs(hir_id, &i.attrs);
         hir::ImplItem {
-            def_id: self.lower_node_id(i.id).expect_owner(),
+            def_id: hir_id.expect_owner(),
             ident: i.ident,
-            attrs: self.lower_attrs(&i.attrs),
             generics,
             vis: self.lower_visibility(&i.vis, None),
             defaultness,
@@ -1024,9 +1046,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_param(&mut self, param: &Param) -> hir::Param<'hir> {
+        let hir_id = self.lower_node_id(param.id);
+        self.lower_attrs(hir_id, &param.attrs);
         hir::Param {
-            attrs: self.lower_attrs(&param.attrs),
-            hir_id: self.lower_node_id(param.id),
+            hir_id,
             pat: self.lower_pat(&param.pat),
             ty_span: param.ty.span,
             span: param.span,
@@ -1158,11 +1181,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 //
                 // If this is the simple case, this parameter will end up being the same as the
                 // original parameter, but with a different pattern id.
-                let mut stmt_attrs = AttrVec::new();
-                stmt_attrs.extend(parameter.attrs.iter().cloned());
+                let stmt_attrs = this.attrs.get(&parameter.hir_id).copied();
                 let (new_parameter_pat, new_parameter_id) = this.pat_ident(desugared_span, ident);
                 let new_parameter = hir::Param {
-                    attrs: parameter.attrs,
                     hir_id: parameter.hir_id,
                     pat: new_parameter_pat,
                     ty_span: parameter.ty_span,
@@ -1205,7 +1226,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     );
                     let move_expr = this.expr_ident(desugared_span, ident, new_parameter_id);
                     let move_stmt = this.stmt_let_pat(
-                        AttrVec::new(),
+                        None,
                         desugared_span,
                         Some(move_expr),
                         move_pat,
@@ -1322,8 +1343,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         match ext {
             Extern::None => abi::Abi::Rust,
             Extern::Implicit => {
-                self.maybe_lint_missing_abi(span, id, abi::Abi::C);
-                abi::Abi::C
+                self.maybe_lint_missing_abi(span, id, abi::Abi::C { unwind: false });
+                abi::Abi::C { unwind: false }
             }
             Extern::Explicit(abi) => self.lower_abi(abi),
         }
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index c564ada0395..44056df4ab9 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -12,7 +12,7 @@
 //! For the simpler lowering steps, IDs and spans should be preserved. Unlike
 //! expansion we do not preserve the process of lowering in the spans, so spans
 //! should not be modified here. When creating a new node (as opposed to
-//! 'folding' an existing one), then you create a new ID using `next_id()`.
+//! "folding" an existing one), create a new ID using `next_id()`.
 //!
 //! You must ensure that IDs are unique. That means that you should only use the
 //! ID from an AST node in a single HIR node (you can assume that AST node-IDs
@@ -26,18 +26,19 @@
 //! span and spans don't need to be kept in order, etc. Where code is preserved
 //! by lowering, it should have the same span as in the AST. Where HIR nodes are
 //! new it is probably best to give a span for the whole AST node being lowered.
-//! All nodes should have real spans, don't use dummy spans. Tools are likely to
+//! All nodes should have real spans; don't use dummy spans. Tools are likely to
 //! get confused if the spans from leaf AST nodes occur in multiple places
 //! in the HIR, especially for multiple identifiers.
 
 #![feature(crate_visibility_modifier)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(box_patterns)]
+#![feature(iter_zip)]
 #![recursion_limit = "256"]
 
 use rustc_ast::node_id::NodeMap;
-use rustc_ast::token::{self, DelimToken, Nonterminal, Token};
-use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, DelimSpan, TokenStream, TokenTree};
+use rustc_ast::token::{self, Token};
+use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream, TokenTree};
 use rustc_ast::visit::{self, AssocCtxt, Visitor};
 use rustc_ast::walk_list;
 use rustc_ast::{self as ast, *};
@@ -55,7 +56,7 @@ use rustc_hir::{ConstArg, GenericArg, ParamName};
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_session::lint::builtin::{BARE_TRAIT_OBJECTS, MISSING_ABI};
 use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer};
-use rustc_session::parse::ParseSess;
+use rustc_session::utils::{FlattenNonterminals, NtToTokenstream};
 use rustc_session::Session;
 use rustc_span::hygiene::ExpnId;
 use rustc_span::source_map::{respan, DesugaringKind};
@@ -92,10 +93,10 @@ struct LoweringContext<'a, 'hir: 'a> {
 
     /// HACK(Centril): there is a cyclic dependency between the parser and lowering
     /// if we don't have this function pointer. To avoid that dependency so that
-    /// librustc_middle is independent of the parser, we use dynamic dispatch here.
+    /// `rustc_middle` is independent of the parser, we use dynamic dispatch here.
     nt_to_tokenstream: NtToTokenstream,
 
-    /// Used to allocate HIR nodes
+    /// Used to allocate HIR nodes.
     arena: &'hir Arena<'hir>,
 
     /// The items being lowered are collected here.
@@ -114,6 +115,8 @@ struct LoweringContext<'a, 'hir: 'a> {
 
     generator_kind: Option<hir::GeneratorKind>,
 
+    attrs: BTreeMap<hir::HirId, &'hir [Attribute]>,
+
     /// When inside an `async` context, this is the `HirId` of the
     /// `task_context` local bound to the resume argument of the generator.
     task_context: Option<hir::HirId>,
@@ -128,7 +131,7 @@ struct LoweringContext<'a, 'hir: 'a> {
     is_in_trait_impl: bool,
     is_in_dyn_type: bool,
 
-    /// What to do when we encounter either an "anonymous lifetime
+    /// What to do when we encounter an "anonymous lifetime
     /// reference". The term "anonymous" is meant to encompass both
     /// `'_` lifetimes as well as fully elided cases where nothing is
     /// written at all (e.g., `&T` or `std::cell::Ref<T>`).
@@ -210,8 +213,6 @@ pub trait ResolverAstLowering {
     ) -> LocalDefId;
 }
 
-type NtToTokenstream = fn(&Nonterminal, &ParseSess, CanSynthesizeMissingTokens) -> TokenStream;
-
 /// Context of `impl Trait` in code, which determines whether it is allowed in an HIR subtree,
 /// and if so, what meaning it has.
 #[derive(Debug)]
@@ -221,7 +222,7 @@ enum ImplTraitContext<'b, 'a> {
     /// equivalent to a fresh universal parameter like `fn foo<T: Debug>(x: T)`.
     ///
     /// Newly generated parameters should be inserted into the given `Vec`.
-    Universal(&'b mut Vec<hir::GenericParam<'a>>),
+    Universal(&'b mut Vec<hir::GenericParam<'a>>, LocalDefId),
 
     /// Treat `impl Trait` as shorthand for a new opaque type.
     /// Example: `fn foo() -> impl Debug`, where `impl Debug` is conceptually
@@ -238,11 +239,13 @@ enum ImplTraitContext<'b, 'a> {
     OtherOpaqueTy {
         /// Set of lifetimes that this opaque type can capture, if it uses
         /// them. This includes lifetimes bound since we entered this context.
-        /// For example, in
+        /// For example:
         ///
+        /// ```
         /// type A<'b> = impl for<'a> Trait<'a, Out = impl Sized + 'a>;
+        /// ```
         ///
-        /// the inner opaque type captures `'a` because it uses it. It doesn't
+        /// Here the inner opaque type captures `'a` because it uses it. It doesn't
         /// need to capture `'b` because it already inherits the lifetime
         /// parameter from `A`.
         // FIXME(impl_trait): but `required_region_bounds` will ICE later
@@ -274,7 +277,7 @@ impl<'a> ImplTraitContext<'_, 'a> {
     fn reborrow<'this>(&'this mut self) -> ImplTraitContext<'this, 'a> {
         use self::ImplTraitContext::*;
         match self {
-            Universal(params) => Universal(params),
+            Universal(params, parent) => Universal(params, *parent),
             ReturnPositionOpaqueTy { fn_def_id, origin } => {
                 ReturnPositionOpaqueTy { fn_def_id: *fn_def_id, origin: *origin }
             }
@@ -307,6 +310,7 @@ pub fn lower_crate<'a, 'hir>(
         bodies: BTreeMap::new(),
         trait_impls: BTreeMap::new(),
         modules: BTreeMap::new(),
+        attrs: BTreeMap::default(),
         exported_macros: Vec::new(),
         non_exported_macro_attrs: Vec::new(),
         catch_scopes: Vec::new(),
@@ -397,67 +401,6 @@ enum AnonymousLifetimeMode {
     PassThrough,
 }
 
-struct TokenStreamLowering<'a> {
-    parse_sess: &'a ParseSess,
-    synthesize_tokens: CanSynthesizeMissingTokens,
-    nt_to_tokenstream: NtToTokenstream,
-}
-
-impl<'a> TokenStreamLowering<'a> {
-    fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
-        tokens.into_trees().flat_map(|tree| self.lower_token_tree(tree).into_trees()).collect()
-    }
-
-    fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
-        match tree {
-            TokenTree::Token(token) => self.lower_token(token),
-            TokenTree::Delimited(span, delim, tts) => {
-                TokenTree::Delimited(span, delim, self.lower_token_stream(tts)).into()
-            }
-        }
-    }
-
-    fn lower_token(&mut self, token: Token) -> TokenStream {
-        match token.kind {
-            token::Interpolated(nt) => {
-                let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
-                TokenTree::Delimited(
-                    DelimSpan::from_single(token.span),
-                    DelimToken::NoDelim,
-                    self.lower_token_stream(tts),
-                )
-                .into()
-            }
-            _ => TokenTree::Token(token).into(),
-        }
-    }
-}
-
-struct ImplTraitTypeIdVisitor<'a> {
-    ids: &'a mut SmallVec<[NodeId; 1]>,
-}
-
-impl Visitor<'_> for ImplTraitTypeIdVisitor<'_> {
-    fn visit_ty(&mut self, ty: &Ty) {
-        match ty.kind {
-            TyKind::Typeof(_) | TyKind::BareFn(_) => return,
-
-            TyKind::ImplTrait(id, _) => self.ids.push(id),
-            _ => {}
-        }
-        visit::walk_ty(self, ty);
-    }
-
-    fn visit_path_segment(&mut self, path_span: Span, path_segment: &PathSegment) {
-        if let Some(ref p) = path_segment.args {
-            if let GenericArgs::Parenthesized(_) = **p {
-                return;
-            }
-        }
-        visit::walk_path_segment(self, path_span, path_segment)
-    }
-}
-
 impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn lower_crate(mut self, c: &Crate) -> hir::Crate<'hir> {
         /// Full-crate AST visitor that inserts into a fresh
@@ -470,25 +413,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         }
 
         impl MiscCollector<'_, '_, '_> {
-            fn allocate_use_tree_hir_id_counters(&mut self, tree: &UseTree, owner: LocalDefId) {
+            fn allocate_use_tree_hir_id_counters(&mut self, tree: &UseTree) {
                 match tree.kind {
                     UseTreeKind::Simple(_, id1, id2) => {
                         for &id in &[id1, id2] {
-                            self.lctx.resolver.create_def(
-                                owner,
-                                id,
-                                DefPathData::Misc,
-                                ExpnId::root(),
-                                tree.prefix.span,
-                            );
                             self.lctx.allocate_hir_id_counter(id);
                         }
                     }
                     UseTreeKind::Glob => (),
                     UseTreeKind::Nested(ref trees) => {
                         for &(ref use_tree, id) in trees {
-                            let hir_id = self.lctx.allocate_hir_id_counter(id);
-                            self.allocate_use_tree_hir_id_counters(use_tree, hir_id.owner);
+                            self.lctx.allocate_hir_id_counter(id);
+                            self.allocate_use_tree_hir_id_counters(use_tree);
                         }
                     }
                 }
@@ -497,7 +433,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         impl<'tcx> Visitor<'tcx> for MiscCollector<'tcx, '_, '_> {
             fn visit_item(&mut self, item: &'tcx Item) {
-                let hir_id = self.lctx.allocate_hir_id_counter(item.id);
+                self.lctx.allocate_hir_id_counter(item.id);
 
                 match item.kind {
                     ItemKind::Struct(_, ref generics)
@@ -516,7 +452,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         self.lctx.type_def_lifetime_params.insert(def_id.to_def_id(), count);
                     }
                     ItemKind::Use(ref use_tree) => {
-                        self.allocate_use_tree_hir_id_counters(use_tree, hir_id.owner);
+                        self.allocate_use_tree_hir_id_counters(use_tree);
                     }
                     _ => {}
                 }
@@ -547,10 +483,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         }
                         self.visit_fn_ret_ty(&f.decl.output)
                     }
-                    TyKind::ImplTrait(def_node_id, _) => {
-                        self.lctx.allocate_hir_id_counter(def_node_id);
-                        visit::walk_ty(self, t);
-                    }
                     _ => visit::walk_ty(self, t),
                 }
             }
@@ -563,7 +495,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         visit::walk_crate(&mut item::ItemLowerer { lctx: &mut self }, c);
 
         let module = self.lower_mod(&c.items, c.span);
-        let attrs = self.lower_attrs(&c.attrs);
+        self.lower_attrs(hir::CRATE_HIR_ID, &c.attrs);
         let body_ids = body_ids(&self.bodies);
         let proc_macros =
             c.proc_macros.iter().map(|id| self.node_id_to_hir_id[*id].unwrap()).collect();
@@ -590,8 +522,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         self.resolver.definitions().init_def_id_to_hir_id_mapping(def_id_to_hir_id);
 
+        #[cfg(debug_assertions)]
+        for (&id, attrs) in self.attrs.iter() {
+            // Verify that we do not store empty slices in the map.
+            if attrs.is_empty() {
+                panic!("Stored empty attributes for {:?}", id);
+            }
+        }
+
         hir::Crate {
-            item: hir::CrateItem { module, attrs, span: c.span },
+            item: module,
             exported_macros: self.arena.alloc_from_iter(self.exported_macros),
             non_exported_macro_attrs: self.arena.alloc_from_iter(self.non_exported_macro_attrs),
             items: self.items,
@@ -604,6 +544,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             modules: self.modules,
             proc_macros,
             trait_map,
+            attrs: self.attrs,
         }
     }
 
@@ -832,7 +773,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         hir::GenericParam {
             hir_id: self.lower_node_id(node_id),
             name: hir_name,
-            attrs: &[],
             bounds: &[],
             span,
             pure_wrt_drop: false,
@@ -926,8 +866,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     // `lifetimes_to_define`. If we swapped the order of these two,
                     // in-band-lifetimes introduced by generics or where-clauses
                     // wouldn't have been added yet.
-                    let generics =
-                        this.lower_generics_mut(generics, ImplTraitContext::Universal(&mut params));
+                    let generics = this.lower_generics_mut(
+                        generics,
+                        ImplTraitContext::Universal(
+                            &mut params,
+                            this.current_hir_id_owner.last().unwrap().0,
+                        ),
+                    );
                     let res = f(this, &mut params);
                     (params, (generics, res))
                 })
@@ -965,11 +910,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         ret
     }
 
-    fn lower_attrs(&mut self, attrs: &[Attribute]) -> &'hir [Attribute] {
-        self.arena.alloc_from_iter(attrs.iter().map(|a| self.lower_attr(a)))
+    fn lower_attrs(&mut self, id: hir::HirId, attrs: &[Attribute]) -> Option<&'hir [Attribute]> {
+        if attrs.is_empty() {
+            None
+        } else {
+            let ret = self.arena.alloc_from_iter(attrs.iter().map(|a| self.lower_attr(a)));
+            debug_assert!(!ret.is_empty());
+            self.attrs.insert(id, ret);
+            Some(ret)
+        }
     }
 
-    fn lower_attr(&mut self, attr: &Attribute) -> Attribute {
+    fn lower_attr(&self, attr: &Attribute) -> Attribute {
         // Note that we explicitly do not walk the path. Since we don't really
         // lower attributes (we use the AST version) there is nowhere to keep
         // the `HirId`s. We don't actually need HIR version of attributes anyway.
@@ -989,7 +941,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         Attribute { kind, id: attr.id, style: attr.style, span: attr.span }
     }
 
-    fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs {
+    fn alias_attrs(&mut self, id: hir::HirId, target_id: hir::HirId) {
+        if let Some(&a) = self.attrs.get(&target_id) {
+            debug_assert!(!a.is_empty());
+            self.attrs.insert(id, a);
+        }
+    }
+
+    fn lower_mac_args(&self, args: &MacArgs) -> MacArgs {
         match *args {
             MacArgs::Empty => MacArgs::Empty,
             MacArgs::Delimited(dspan, delim, ref tokens) => {
@@ -1040,12 +999,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     }
                 }
 
-                let tokens = TokenStreamLowering {
+                let tokens = FlattenNonterminals {
                     parse_sess: &self.sess.parse_sess,
                     synthesize_tokens: CanSynthesizeMissingTokens::Yes,
                     nt_to_tokenstream: self.nt_to_tokenstream,
                 }
-                .lower_token(token.clone());
+                .process_token(token.clone());
                 MacArgs::Eq(eq_span, unwrap_single_token(self.sess, tokens, token.span))
             }
         }
@@ -1056,12 +1015,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         tokens: TokenStream,
         synthesize_tokens: CanSynthesizeMissingTokens,
     ) -> TokenStream {
-        TokenStreamLowering {
+        FlattenNonterminals {
             parse_sess: &self.sess.parse_sess,
             synthesize_tokens,
             nt_to_tokenstream: self.nt_to_tokenstream,
         }
-        .lower_token_stream(tokens)
+        .process_token_stream(tokens)
     }
 
     /// Given an associated type constraint like one of these:
@@ -1118,6 +1077,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
             AssocTyConstraintKind::Bound { ref bounds } => {
                 let mut capturable_lifetimes;
+                let mut parent_def_id = self.current_hir_id_owner.last().unwrap().0;
                 // Piggy-back on the `impl Trait` context to figure out the correct behavior.
                 let (desugar_to_impl_trait, itctx) = match itctx {
                     // We are in the return position:
@@ -1137,7 +1097,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     // so desugar to
                     //
                     //     fn foo(x: dyn Iterator<Item = impl Debug>)
-                    ImplTraitContext::Universal(..) if self.is_in_dyn_type => (true, itctx),
+                    ImplTraitContext::Universal(_, parent) if self.is_in_dyn_type => {
+                        parent_def_id = parent;
+                        (true, itctx)
+                    }
 
                     // In `type Foo = dyn Iterator<Item: Debug>` we desugar to
                     // `type Foo = dyn Iterator<Item = impl Debug>` but we have to override the
@@ -1171,7 +1134,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     // constructing the HIR for `impl bounds...` and then lowering that.
 
                     let impl_trait_node_id = self.resolver.next_node_id();
-                    let parent_def_id = self.current_hir_id_owner.last().unwrap().0;
                     self.resolver.create_def(
                         parent_def_id,
                         impl_trait_node_id,
@@ -1393,7 +1355,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 if kind != TraitObjectSyntax::Dyn {
                     self.maybe_lint_bare_trait(t.span, t.id, false);
                 }
-                hir::TyKind::TraitObject(bounds, lifetime_bound)
+                hir::TyKind::TraitObject(bounds, lifetime_bound, kind)
             }
             TyKind::ImplTrait(def_node_id, ref bounds) => {
                 let span = t.span;
@@ -1424,25 +1386,20 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             |this| this.lower_param_bounds(bounds, nested_itctx),
                         )
                     }
-                    ImplTraitContext::Universal(in_band_ty_params) => {
+                    ImplTraitContext::Universal(in_band_ty_params, parent_def_id) => {
                         // Add a definition for the in-band `Param`.
                         let def_id = self.resolver.local_def_id(def_node_id);
 
-                        self.allocate_hir_id_counter(def_node_id);
-
-                        let hir_bounds = self.with_hir_id_owner(def_node_id, |this| {
-                            this.lower_param_bounds(
-                                bounds,
-                                ImplTraitContext::Universal(in_band_ty_params),
-                            )
-                        });
+                        let hir_bounds = self.lower_param_bounds(
+                            bounds,
+                            ImplTraitContext::Universal(in_band_ty_params, parent_def_id),
+                        );
                         // Set the name to `impl Bound1 + Bound2`.
                         let ident = Ident::from_str_and_span(&pprust::ty_to_string(t), span);
                         in_band_ty_params.push(hir::GenericParam {
                             hir_id: self.lower_node_id(def_node_id),
                             name: ParamName::Plain(ident),
                             pure_wrt_drop: false,
-                            attrs: &[],
                             bounds: hir_bounds,
                             span,
                             kind: hir::GenericParamKind::Type {
@@ -1570,7 +1527,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let opaque_ty_item = hir::Item {
             def_id: opaque_ty_id,
             ident: Ident::invalid(),
-            attrs: Default::default(),
             kind: opaque_ty_item_kind,
             vis: respan(span.shrink_to_lo(), hir::VisibilityKind::Inherited),
             span: opaque_ty_span,
@@ -1731,7 +1687,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         name,
                         span: lifetime.span,
                         pure_wrt_drop: false,
-                        attrs: &[],
                         bounds: &[],
                         kind: hir::GenericParamKind::Lifetime { kind },
                     });
@@ -1764,14 +1719,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         )
     }
 
-    fn lower_local(&mut self, l: &Local) -> (hir::Local<'hir>, SmallVec<[NodeId; 1]>) {
-        let mut ids = SmallVec::<[NodeId; 1]>::new();
-        if self.sess.features_untracked().impl_trait_in_bindings {
-            if let Some(ref ty) = l.ty {
-                let mut visitor = ImplTraitTypeIdVisitor { ids: &mut ids };
-                visitor.visit_ty(ty);
-            }
-        }
+    fn lower_local(&mut self, l: &Local) -> hir::Local<'hir> {
         let ty = l.ty.as_ref().map(|t| {
             let mut capturable_lifetimes;
             self.lower_ty(
@@ -1788,18 +1736,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             )
         });
         let init = l.init.as_ref().map(|e| self.lower_expr(e));
-        (
-            hir::Local {
-                hir_id: self.lower_node_id(l.id),
-                ty,
-                pat: self.lower_pat(&l.pat),
-                init,
-                span: l.span,
-                attrs: l.attrs.iter().map(|a| self.lower_attr(a)).collect::<Vec<_>>().into(),
-                source: hir::LocalSource::Normal,
-            },
-            ids,
-        )
+        let hir_id = self.lower_node_id(l.id);
+        self.lower_attrs(hir_id, &l.attrs);
+        hir::Local {
+            hir_id,
+            ty,
+            pat: self.lower_pat(&l.pat),
+            init,
+            span: l.span,
+            source: hir::LocalSource::Normal,
+        }
     }
 
     fn lower_fn_params_to_names(&mut self, decl: &FnDecl) -> &'hir [Ident] {
@@ -1866,7 +1812,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
             this.arena.alloc_from_iter(inputs.iter().map(|param| {
                 if let Some((_, ibty)) = &mut in_band_ty_params {
-                    this.lower_ty_direct(&param.ty, ImplTraitContext::Universal(ibty))
+                    this.lower_ty_direct(
+                        &param.ty,
+                        ImplTraitContext::Universal(
+                            ibty,
+                            this.current_hir_id_owner.last().unwrap().0,
+                        ),
+                    )
                 } else {
                     this.lower_ty_direct(&param.ty, ImplTraitContext::disallowed())
                 }
@@ -2110,7 +2062,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         hir::FnRetTy::Return(self.arena.alloc(opaque_ty))
     }
 
-    /// Transforms `-> T` into `Future<Output = T>`
+    /// Transforms `-> T` into `Future<Output = T>`.
     fn lower_async_fn_output_type_to_future_bound(
         &mut self,
         output: &FnRetTy,
@@ -2269,13 +2221,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                 let kind = hir::GenericParamKind::Type {
                     default: default.as_ref().map(|x| {
-                        self.lower_ty(
-                            x,
-                            ImplTraitContext::OtherOpaqueTy {
-                                capturable_lifetimes: &mut FxHashSet::default(),
-                                origin: hir::OpaqueTyOrigin::Misc,
-                            },
-                        )
+                        self.lower_ty(x, ImplTraitContext::Disallowed(ImplTraitPosition::Other))
                     }),
                     synthetic: param
                         .attrs
@@ -2293,17 +2239,17 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         this.lower_ty(&ty, ImplTraitContext::disallowed())
                     });
                 let default = default.as_ref().map(|def| self.lower_anon_const(def));
-
                 (hir::ParamName::Plain(param.ident), hir::GenericParamKind::Const { ty, default })
             }
         };
 
+        let hir_id = self.lower_node_id(param.id);
+        self.lower_attrs(hir_id, &param.attrs);
         hir::GenericParam {
-            hir_id: self.lower_node_id(param.id),
+            hir_id,
             name,
             span: param.ident.span,
             pure_wrt_drop: self.sess.contains_name(&param.attrs, sym::may_dangle),
-            attrs: self.lower_attrs(&param.attrs),
             bounds: self.arena.alloc_from_iter(bounds),
             kind,
         }
@@ -2383,26 +2329,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     }
 
     fn lower_block_noalloc(&mut self, b: &Block, targeted_by_break: bool) -> hir::Block<'hir> {
-        let mut expr: Option<&'hir _> = None;
-
-        let stmts = self.arena.alloc_from_iter(
-            b.stmts
-                .iter()
-                .enumerate()
-                .filter_map(|(index, stmt)| {
-                    if index == b.stmts.len() - 1 {
-                        if let StmtKind::Expr(ref e) = stmt.kind {
-                            expr = Some(self.lower_expr(e));
-                            None
-                        } else {
-                            Some(self.lower_stmt(stmt))
-                        }
-                    } else {
-                        Some(self.lower_stmt(stmt))
-                    }
-                })
-                .flatten(),
-        );
+        let (stmts, expr) = match &*b.stmts {
+            [stmts @ .., Stmt { kind: StmtKind::Expr(e), .. }] => (stmts, Some(&*e)),
+            stmts => (stmts, None),
+        };
+        let stmts = self.arena.alloc_from_iter(stmts.iter().flat_map(|stmt| self.lower_stmt(stmt)));
+        let expr = expr.map(|e| self.lower_expr(e));
         let rules = self.lower_block_check_mode(&b.rules);
         let hir_id = self.lower_node_id(b.id);
 
@@ -2424,27 +2356,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     }
 
     fn lower_stmt(&mut self, s: &Stmt) -> SmallVec<[hir::Stmt<'hir>; 1]> {
-        let kind = match s.kind {
+        let (hir_id, kind) = match s.kind {
             StmtKind::Local(ref l) => {
-                let (l, item_ids) = self.lower_local(l);
-                let mut ids: SmallVec<[hir::Stmt<'hir>; 1]> = item_ids
-                    .into_iter()
-                    .map(|item_id| {
-                        let item_id = hir::ItemId {
-                            // All the items that `lower_local` finds are `impl Trait` types.
-                            def_id: self.lower_node_id(item_id).expect_owner(),
-                        };
-                        self.stmt(s.span, hir::StmtKind::Item(item_id))
-                    })
-                    .collect();
-                ids.push({
-                    hir::Stmt {
-                        hir_id: self.lower_node_id(s.id),
-                        kind: hir::StmtKind::Local(self.arena.alloc(l)),
-                        span: s.span,
-                    }
-                });
-                return ids;
+                let l = self.lower_local(l);
+                let hir_id = self.lower_node_id(s.id);
+                self.alias_attrs(hir_id, l.hir_id);
+                return smallvec![hir::Stmt {
+                    hir_id,
+                    kind: hir::StmtKind::Local(self.arena.alloc(l)),
+                    span: s.span,
+                }];
             }
             StmtKind::Item(ref it) => {
                 // Can only use the ID once.
@@ -2462,12 +2383,22 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     })
                     .collect();
             }
-            StmtKind::Expr(ref e) => hir::StmtKind::Expr(self.lower_expr(e)),
-            StmtKind::Semi(ref e) => hir::StmtKind::Semi(self.lower_expr(e)),
+            StmtKind::Expr(ref e) => {
+                let e = self.lower_expr(e);
+                let hir_id = self.lower_node_id(s.id);
+                self.alias_attrs(hir_id, e.hir_id);
+                (hir_id, hir::StmtKind::Expr(e))
+            }
+            StmtKind::Semi(ref e) => {
+                let e = self.lower_expr(e);
+                let hir_id = self.lower_node_id(s.id);
+                self.alias_attrs(hir_id, e.hir_id);
+                (hir_id, hir::StmtKind::Semi(e))
+            }
             StmtKind::Empty => return smallvec![],
             StmtKind::MacCall(..) => panic!("shouldn't exist here"),
         };
-        smallvec![hir::Stmt { hir_id: self.lower_node_id(s.id), kind, span: s.span }]
+        smallvec![hir::Stmt { hir_id, kind, span: s.span }]
     }
 
     fn lower_block_check_mode(&mut self, b: &BlockCheckMode) -> hir::BlockCheckMode {
@@ -2511,13 +2442,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
     fn stmt_let_pat(
         &mut self,
-        attrs: AttrVec,
+        attrs: Option<&'hir [Attribute]>,
         span: Span,
         init: Option<&'hir hir::Expr<'hir>>,
         pat: &'hir hir::Pat<'hir>,
         source: hir::LocalSource,
     ) -> hir::Stmt<'hir> {
-        let local = hir::Local { attrs, hir_id: self.next_id(), init, pat, source, span, ty: None };
+        let hir_id = self.next_id();
+        if let Some(a) = attrs {
+            debug_assert!(!a.is_empty());
+            self.attrs.insert(hir_id, a);
+        }
+        let local = hir::Local { hir_id, init, pat, source, span, ty: None };
         self.stmt(span, hir::StmtKind::Local(self.arena.alloc(local)))
     }
 
@@ -2571,8 +2507,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         &mut self,
         span: Span,
         pat: &'hir hir::Pat<'hir>,
-    ) -> &'hir [hir::FieldPat<'hir>] {
-        let field = hir::FieldPat {
+    ) -> &'hir [hir::PatField<'hir>] {
+        let field = hir::PatField {
             hir_id: self.next_id(),
             ident: Ident::new(sym::integer(0), span),
             is_shorthand: false,
@@ -2586,7 +2522,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         &mut self,
         span: Span,
         lang_item: hir::LangItem,
-        fields: &'hir [hir::FieldPat<'hir>],
+        fields: &'hir [hir::PatField<'hir>],
     ) -> &'hir hir::Pat<'hir> {
         let qpath = hir::QPath::LangItem(lang_item, span);
         self.pat(span, hir::PatKind::Struct(qpath, fields, false))
@@ -2660,6 +2596,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         hir::TyKind::TraitObject(
                             arena_vec![self; principal],
                             self.elided_dyn_bound(span),
+                            TraitObjectSyntax::None,
                         )
                     }
                     _ => hir::TyKind::Path(hir::QPath::Resolved(None, path)),
diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs
index e4e7b24d29e..2451409aac8 100644
--- a/compiler/rustc_ast_lowering/src/pat.rs
+++ b/compiler/rustc_ast_lowering/src/pat.rs
@@ -56,7 +56,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             ImplTraitContext::disallowed(),
                         );
 
-                        let fs = self.arena.alloc_from_iter(fields.iter().map(|f| hir::FieldPat {
+                        let fs = self.arena.alloc_from_iter(fields.iter().map(|f| hir::PatField {
                             hir_id: self.next_id(),
                             ident: f.ident,
                             pat: self.lower_pat(&f.pat),
diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs
index cb4d5ea6ee6..46dac2f1af4 100644
--- a/compiler/rustc_ast_lowering/src/path.rs
+++ b/compiler/rustc_ast_lowering/src/path.rs
@@ -30,6 +30,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let partial_res =
             self.resolver.get_partial_res(id).unwrap_or_else(|| PartialRes::new(Res::Err));
 
+        let path_span_lo = p.span.shrink_to_lo();
         let proj_start = p.segments.len() - partial_res.unresolved_segments();
         let path = self.arena.alloc(hir::Path {
             res: self.lower_res(partial_res.base_res()),
@@ -108,7 +109,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     )
                 },
             )),
-            span: p.span,
+            span: p.segments[..proj_start]
+                .last()
+                .map_or(path_span_lo, |segment| path_span_lo.to(segment.span())),
         });
 
         // Simple case, either no projections, or only fully-qualified.
@@ -127,7 +130,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             // e.g., `Vec` in `Vec::new` or `<I as Iterator>::Item` in
             // `<I as Iterator>::Item::default`.
             let new_id = self.next_id();
-            self.arena.alloc(self.ty_path(new_id, p.span, hir::QPath::Resolved(qself, path)))
+            self.arena.alloc(self.ty_path(new_id, path.span, hir::QPath::Resolved(qself, path)))
         };
 
         // Anything after the base path are associated "extensions",
@@ -141,7 +144,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         //   3. `<<std::vec::Vec<T>>::IntoIter>::Item`
         // * final path is `<<<std::vec::Vec<T>>::IntoIter>::Item>::clone`
         for (i, segment) in p.segments.iter().enumerate().skip(proj_start) {
-            let segment = self.arena.alloc(self.lower_path_segment(
+            let hir_segment = self.arena.alloc(self.lower_path_segment(
                 p.span,
                 segment,
                 param_mode,
@@ -150,7 +153,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 itctx.reborrow(),
                 None,
             ));
-            let qpath = hir::QPath::TypeRelative(ty, segment);
+            let qpath = hir::QPath::TypeRelative(ty, hir_segment);
 
             // It's finished, return the extension of the right node type.
             if i == p.segments.len() - 1 {
@@ -159,7 +162,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
             // Wrap the associated extension in another type node.
             let new_id = self.next_id();
-            ty = self.arena.alloc(self.ty_path(new_id, p.span, qpath));
+            ty = self.arena.alloc(self.ty_path(new_id, path_span_lo.to(segment.span()), qpath));
         }
 
         // We should've returned in the for loop above.
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 563bcda5190..8a3f7641596 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -532,6 +532,25 @@ impl<'a> AstValidator<'a> {
         }
     }
 
+    /// An item in `extern { ... }` cannot use non-ascii identifier.
+    fn check_foreign_item_ascii_only(&self, ident: Ident) {
+        let symbol_str = ident.as_str();
+        if !symbol_str.is_ascii() {
+            let n = 83942;
+            self.err_handler()
+                .struct_span_err(
+                    ident.span,
+                    "items in `extern` blocks cannot use non-ascii identifiers",
+                )
+                .span_label(self.current_extern_span(), "in this `extern` block")
+                .note(&format!(
+                    "This limitation may be lifted in the future; see issue #{} <https://github.com/rust-lang/rust/issues/{}> for more information",
+                    n, n,
+                ))
+                .emit();
+        }
+    }
+
     /// Reject C-varadic type unless the function is foreign,
     /// or free and `unsafe extern "C"` semantically.
     fn check_c_varadic_type(&self, fk: FnKind<'a>) {
@@ -592,7 +611,7 @@ impl<'a> AstValidator<'a> {
             self.session,
             ident.span,
             E0754,
-            "trying to load file for module `{}` with non ascii identifer name",
+            "trying to load file for module `{}` with non-ascii identifier name",
             ident.name
         )
         .help("consider using `#[path]` attribute to specify filesystem path")
@@ -1103,15 +1122,18 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 self.check_defaultness(fi.span, *def);
                 self.check_foreign_fn_bodyless(fi.ident, body.as_deref());
                 self.check_foreign_fn_headerless(fi.ident, fi.span, sig.header);
+                self.check_foreign_item_ascii_only(fi.ident);
             }
             ForeignItemKind::TyAlias(box TyAliasKind(def, generics, bounds, body)) => {
                 self.check_defaultness(fi.span, *def);
                 self.check_foreign_kind_bodyless(fi.ident, "type", body.as_ref().map(|b| b.span));
                 self.check_type_no_bounds(bounds, "`extern` blocks");
                 self.check_foreign_ty_genericless(generics);
+                self.check_foreign_item_ascii_only(fi.ident);
             }
             ForeignItemKind::Static(_, _, body) => {
                 self.check_foreign_kind_bodyless(fi.ident, "static", body.as_ref().map(|b| b.span));
+                self.check_foreign_item_ascii_only(fi.ident);
             }
             ForeignItemKind::MacCall(..) => {}
         }
@@ -1150,20 +1172,23 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
     }
 
     fn visit_generics(&mut self, generics: &'a Generics) {
-        let mut prev_ty_default = None;
+        let cg_defaults = self.session.features_untracked().const_generics_defaults;
+
+        let mut prev_param_default = None;
         for param in &generics.params {
             match param.kind {
                 GenericParamKind::Lifetime => (),
-                GenericParamKind::Type { default: Some(_), .. } => {
-                    prev_ty_default = Some(param.ident.span);
+                GenericParamKind::Type { default: Some(_), .. }
+                | GenericParamKind::Const { default: Some(_), .. } => {
+                    prev_param_default = Some(param.ident.span);
                 }
                 GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {
-                    if let Some(span) = prev_ty_default {
+                    if let Some(span) = prev_param_default {
                         let mut err = self.err_handler().struct_span_err(
                             span,
-                            "type parameters with a default must be trailing",
+                            "generic parameters with a default must be trailing",
                         );
-                        if matches!(param.kind, GenericParamKind::Const { .. }) {
+                        if matches!(param.kind, GenericParamKind::Const { .. }) && !cg_defaults {
                             err.note(
                                 "using type defaults and const parameters \
                                  in the same parameter list is currently not permitted",
@@ -1188,8 +1213,41 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 deny_equality_constraints(self, predicate, generics);
             }
         }
-
-        visit::walk_generics(self, generics)
+        walk_list!(self, visit_generic_param, &generics.params);
+        for predicate in &generics.where_clause.predicates {
+            match predicate {
+                WherePredicate::BoundPredicate(bound_pred) => {
+                    // A type binding, eg `for<'c> Foo: Send+Clone+'c`
+                    self.check_late_bound_lifetime_defs(&bound_pred.bound_generic_params);
+
+                    // This is slightly complicated. Our representation for poly-trait-refs contains a single
+                    // binder and thus we only allow a single level of quantification. However,
+                    // the syntax of Rust permits quantification in two places in where clauses,
+                    // e.g., `T: for <'a> Foo<'a>` and `for <'a, 'b> &'b T: Foo<'a>`. If both are
+                    // defined, then error.
+                    if !bound_pred.bound_generic_params.is_empty() {
+                        for bound in &bound_pred.bounds {
+                            match bound {
+                                GenericBound::Trait(t, _) => {
+                                    if !t.bound_generic_params.is_empty() {
+                                        struct_span_err!(
+                                            self.err_handler(),
+                                            t.span,
+                                            E0316,
+                                            "nested quantification of lifetimes"
+                                        )
+                                        .emit();
+                                    }
+                                }
+                                GenericBound::Outlives(_) => {}
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+            self.visit_where_predicate(predicate);
+        }
     }
 
     fn visit_generic_param(&mut self, param: &'a GenericParam) {
@@ -1238,14 +1296,6 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         visit::walk_pat(self, pat)
     }
 
-    fn visit_where_predicate(&mut self, p: &'a WherePredicate) {
-        if let &WherePredicate::BoundPredicate(ref bound_predicate) = p {
-            // A type binding, eg `for<'c> Foo: Send+Clone+'c`
-            self.check_late_bound_lifetime_defs(&bound_predicate.bound_generic_params);
-        }
-        visit::walk_where_predicate(self, p);
-    }
-
     fn visit_poly_trait_ref(&mut self, t: &'a PolyTraitRef, m: &'a TraitBoundModifier) {
         self.check_late_bound_lifetime_defs(&t.bound_generic_params);
         visit::walk_poly_trait_ref(self, t, m);
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index 474ec2b589b..0679ccabe72 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -8,7 +8,7 @@ use rustc_feature::{Features, GateIssue};
 use rustc_session::parse::{feature_err, feature_err_issue};
 use rustc_session::Session;
 use rustc_span::source_map::Spanned;
-use rustc_span::symbol::{sym, Symbol};
+use rustc_span::symbol::sym;
 use rustc_span::Span;
 
 use tracing::debug;
@@ -164,6 +164,46 @@ impl<'a> PostExpansionVisitor<'a> {
                     "C-cmse-nonsecure-call ABI is experimental and subject to change"
                 );
             }
+            "C-unwind" => {
+                gate_feature_post!(
+                    &self,
+                    c_unwind,
+                    span,
+                    "C-unwind ABI is experimental and subject to change"
+                );
+            }
+            "stdcall-unwind" => {
+                gate_feature_post!(
+                    &self,
+                    c_unwind,
+                    span,
+                    "stdcall-unwind ABI is experimental and subject to change"
+                );
+            }
+            "system-unwind" => {
+                gate_feature_post!(
+                    &self,
+                    c_unwind,
+                    span,
+                    "system-unwind ABI is experimental and subject to change"
+                );
+            }
+            "thiscall-unwind" => {
+                gate_feature_post!(
+                    &self,
+                    c_unwind,
+                    span,
+                    "thiscall-unwind ABI is experimental and subject to change"
+                );
+            }
+            "wasm" => {
+                gate_feature_post!(
+                    &self,
+                    wasm_abi,
+                    span,
+                    "wasm ABI is experimental and subject to change"
+                );
+            }
             abi => self
                 .sess
                 .parse_sess
@@ -247,7 +287,7 @@ impl<'a> PostExpansionVisitor<'a> {
                 if let ast::TyKind::ImplTrait(..) = ty.kind {
                     gate_feature_post!(
                         &self.vis,
-                        type_alias_impl_trait,
+                        min_type_alias_impl_trait,
                         ty.span,
                         "`impl Trait` in type aliases is unstable"
                     );
@@ -281,24 +321,13 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                     include => external_doc
                     cfg => doc_cfg
                     masked => doc_masked
-                    spotlight => doc_spotlight
+                    notable_trait => doc_notable_trait
                     keyword => doc_keyword
                 );
             }
         }
     }
 
-    fn visit_name(&mut self, sp: Span, name: Symbol) {
-        if !name.as_str().is_ascii() {
-            gate_feature_post!(
-                &self,
-                non_ascii_idents,
-                self.sess.parse_sess.source_map().guess_head_span(sp),
-                "non-ascii idents are not fully supported"
-            );
-        }
-    }
-
     fn visit_item(&mut self, i: &'a ast::Item) {
         match i.kind {
             ast::ItemKind::ForeignMod(ref foreign_module) => {
@@ -326,16 +355,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                          over time"
                     );
                 }
-                if self.sess.contains_name(&i.attrs[..], sym::main) {
-                    gate_feature_post!(
-                        &self,
-                        main,
-                        i.span,
-                        "declaration of a non-standard `#[main]` \
-                         function may change over time, for now \
-                         a top-level `fn main()` is required"
-                    );
-                }
             }
 
             ast::ItemKind::Struct(..) => {
@@ -638,15 +657,22 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
             }
         };
     }
-    gate_all!(if_let_guard, "`if let` guards are experimental");
-    gate_all!(let_chains, "`let` expressions in this position are experimental");
+    gate_all!(
+        if_let_guard,
+        "`if let` guards are experimental",
+        "you can write `if matches!(<expr>, <pattern>)` instead of `if let <pattern> = <expr>`"
+    );
+    gate_all!(
+        let_chains,
+        "`let` expressions in this position are experimental",
+        "you can write `matches!(<expr>, <pattern>)` instead of `let <pattern> = <expr>`"
+    );
     gate_all!(
         async_closure,
         "async closures are unstable",
         "to use an async block, remove the `||`: `async {`"
     );
     gate_all!(generators, "yield syntax is experimental");
-    gate_all!(or_patterns, "or-patterns syntax is experimental");
     gate_all!(raw_ref_op, "raw address of syntax is experimental");
     gate_all!(const_trait_bound_opt_out, "`?const` on trait bounds is experimental");
     gate_all!(const_trait_impl, "const trait impls are experimental");
@@ -701,16 +727,46 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
 }
 
 fn maybe_stage_features(sess: &Session, krate: &ast::Crate) {
+    use rustc_errors::Applicability;
+
     if !sess.opts.unstable_features.is_nightly_build() {
+        let lang_features = &sess.features_untracked().declared_lang_features;
         for attr in krate.attrs.iter().filter(|attr| sess.check_name(attr, sym::feature)) {
-            struct_span_err!(
+            let mut err = struct_span_err!(
                 sess.parse_sess.span_diagnostic,
                 attr.span,
                 E0554,
                 "`#![feature]` may not be used on the {} release channel",
                 option_env!("CFG_RELEASE_CHANNEL").unwrap_or("(unknown)")
-            )
-            .emit();
+            );
+            let mut all_stable = true;
+            for ident in
+                attr.meta_item_list().into_iter().flatten().map(|nested| nested.ident()).flatten()
+            {
+                let name = ident.name;
+                let stable_since = lang_features
+                    .iter()
+                    .flat_map(|&(feature, _, since)| if feature == name { since } else { None })
+                    .next();
+                if let Some(since) = stable_since {
+                    err.help(&format!(
+                        "the feature `{}` has been stable since {} and no longer requires \
+                                  an attribute to enable",
+                        name, since
+                    ));
+                } else {
+                    all_stable = false;
+                }
+            }
+            if all_stable {
+                err.span_suggestion(
+                    attr.span,
+                    "remove the attribute",
+                    String::new(),
+                    Applicability::MachineApplicable,
+                );
+            }
+            err.emit();
         }
     }
 }
diff --git a/compiler/rustc_ast_passes/src/node_count.rs b/compiler/rustc_ast_passes/src/node_count.rs
index fb7e0d3450f..3980e6da682 100644
--- a/compiler/rustc_ast_passes/src/node_count.rs
+++ b/compiler/rustc_ast_passes/src/node_count.rs
@@ -88,9 +88,9 @@ impl<'ast> Visitor<'ast> for NodeCounter {
         self.count += 1;
         walk_struct_def(self, s)
     }
-    fn visit_struct_field(&mut self, s: &StructField) {
+    fn visit_field_def(&mut self, s: &FieldDef) {
         self.count += 1;
-        walk_struct_field(self, s)
+        walk_field_def(self, s)
     }
     fn visit_enum_def(
         &mut self,
diff --git a/compiler/rustc_ast_pretty/src/lib.rs b/compiler/rustc_ast_pretty/src/lib.rs
index d869baad012..67b66284f66 100644
--- a/compiler/rustc_ast_pretty/src/lib.rs
+++ b/compiler/rustc_ast_pretty/src/lib.rs
@@ -1,6 +1,6 @@
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(box_patterns)]
 #![recursion_limit = "256"]
 
diff --git a/compiler/rustc_ast_pretty/src/pprust/mod.rs b/compiler/rustc_ast_pretty/src/pprust/mod.rs
index b88699f6ee1..976725b308e 100644
--- a/compiler/rustc_ast_pretty/src/pprust/mod.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/mod.rs
@@ -22,10 +22,6 @@ pub fn token_to_string(token: &Token) -> String {
     State::new().token_to_string(token)
 }
 
-pub fn token_to_string_ext(token: &Token, convert_dollar_crate: bool) -> String {
-    State::new().token_to_string_ext(token, convert_dollar_crate)
-}
-
 pub fn ty_to_string(ty: &ast::Ty) -> String {
     State::new().ty_to_string(ty)
 }
@@ -50,18 +46,10 @@ pub fn tts_to_string(tokens: &TokenStream) -> String {
     State::new().tts_to_string(tokens)
 }
 
-pub fn stmt_to_string(stmt: &ast::Stmt) -> String {
-    State::new().stmt_to_string(stmt)
-}
-
 pub fn item_to_string(i: &ast::Item) -> String {
     State::new().item_to_string(i)
 }
 
-pub fn generic_params_to_string(generic_params: &[ast::GenericParam]) -> String {
-    State::new().generic_params_to_string(generic_params)
-}
-
 pub fn path_to_string(p: &ast::Path) -> String {
     State::new().path_to_string(p)
 }
@@ -74,26 +62,14 @@ pub fn vis_to_string(v: &ast::Visibility) -> String {
     State::new().vis_to_string(v)
 }
 
-pub fn block_to_string(blk: &ast::Block) -> String {
-    State::new().block_to_string(blk)
-}
-
 pub fn meta_list_item_to_string(li: &ast::NestedMetaItem) -> String {
     State::new().meta_list_item_to_string(li)
 }
 
-pub fn attr_item_to_string(ai: &ast::AttrItem) -> String {
-    State::new().attr_item_to_string(ai)
-}
-
 pub fn attribute_to_string(attr: &ast::Attribute) -> String {
     State::new().attribute_to_string(attr)
 }
 
-pub fn param_to_string(arg: &ast::Param) -> String {
-    State::new().param_to_string(arg)
-}
-
 pub fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
     State::new().to_string(f)
 }
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 82f6e936b76..789d2c296e2 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -1711,7 +1711,7 @@ impl<'a> State<'a> {
     fn print_expr_struct(
         &mut self,
         path: &ast::Path,
-        fields: &[ast::Field],
+        fields: &[ast::ExprField],
         rest: &ast::StructRest,
         attrs: &[ast::Attribute],
     ) {
@@ -1873,8 +1873,8 @@ impl<'a> State<'a> {
             ast::ExprKind::Repeat(ref element, ref count) => {
                 self.print_expr_repeat(element, count, attrs);
             }
-            ast::ExprKind::Struct(ref path, ref fields, ref rest) => {
-                self.print_expr_struct(path, &fields[..], rest, attrs);
+            ast::ExprKind::Struct(ref se) => {
+                self.print_expr_struct(&se.path, &se.fields, &se.rest, attrs);
             }
             ast::ExprKind::Tup(ref exprs) => {
                 self.print_expr_tup(&exprs[..], attrs);
@@ -2149,10 +2149,10 @@ impl<'a> State<'a> {
                                     None => s.word("_"),
                                 }
                             }
-                            InlineAsmOperand::Const { expr } => {
+                            InlineAsmOperand::Const { anon_const } => {
                                 s.word("const");
                                 s.space();
-                                s.print_expr(expr);
+                                s.print_expr(&anon_const.value);
                             }
                             InlineAsmOperand::Sym { expr } => {
                                 s.word("sym");
@@ -2292,10 +2292,6 @@ impl<'a> State<'a> {
         }
     }
 
-    pub fn print_usize(&mut self, i: usize) {
-        self.s.word(i.to_string())
-    }
-
     crate fn print_name(&mut self, name: Symbol) {
         self.s.word(name.to_string());
         self.ann.post(self, AnnNode::Name(&name))
@@ -2659,8 +2655,10 @@ impl<'a> State<'a> {
                     s.word_space(":");
                     s.print_type(ty);
                     s.print_type_bounds(":", &param.bounds);
-                    if let Some(ref _default) = default {
-                        // FIXME(const_generics_defaults): print the `default` value here
+                    if let Some(ref default) = default {
+                        s.s.space();
+                        s.word_space("=");
+                        s.print_expr(&default.value);
                     }
                 }
             }
diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index e58b266fdb9..20971ebb957 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -862,18 +862,6 @@ pub fn find_repr_attrs(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
         if let Some(items) = attr.meta_item_list() {
             sess.mark_attr_used(attr);
             for item in items {
-                if !item.is_meta_item() {
-                    handle_errors(
-                        &sess.parse_sess,
-                        item.span(),
-                        AttrError::UnsupportedLiteral(
-                            "meta item in `repr` must be an identifier",
-                            false,
-                        ),
-                    );
-                    continue;
-                }
-
                 let mut recognised = false;
                 if item.is_word() {
                     let hint = match item.name_or_empty() {
@@ -890,23 +878,6 @@ pub fn find_repr_attrs(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         acc.push(h);
                     }
                 } else if let Some((name, value)) = item.name_value_literal() {
-                    let parse_alignment = |node: &ast::LitKind| -> Result<u32, &'static str> {
-                        if let ast::LitKind::Int(literal, ast::LitIntType::Unsuffixed) = node {
-                            if literal.is_power_of_two() {
-                                // rustc_middle::ty::layout::Align restricts align to <= 2^29
-                                if *literal <= 1 << 29 {
-                                    Ok(*literal as u32)
-                                } else {
-                                    Err("larger than 2^29")
-                                }
-                            } else {
-                                Err("not a power of two")
-                            }
-                        } else {
-                            Err("not an unsuffixed integer")
-                        }
-                    };
-
                     let mut literal_error = None;
                     if name == sym::align {
                         recognised = true;
@@ -966,13 +937,7 @@ pub fn find_repr_attrs(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                 }
                 if !recognised {
                     // Not a word we recognize
-                    struct_span_err!(
-                        diagnostic,
-                        item.span(),
-                        E0552,
-                        "unrecognized representation hint"
-                    )
-                    .emit();
+                    diagnostic.delay_span_bug(item.span(), "unrecognized representation hint");
                 }
             }
         }
@@ -1080,3 +1045,16 @@ fn allow_unstable<'a>(
         name
     })
 }
+
+pub fn parse_alignment(node: &ast::LitKind) -> Result<u32, &'static str> {
+    if let ast::LitKind::Int(literal, ast::LitIntType::Unsuffixed) = node {
+        if literal.is_power_of_two() {
+            // rustc_middle::ty::layout::Align restricts align to <= 2^29
+            if *literal <= 1 << 29 { Ok(*literal as u32) } else { Err("larger than 2^29") }
+        } else {
+            Err("not a power of two")
+        }
+    } else {
+        Err("not an unsuffixed integer")
+    }
+}
diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs
index 149a950f7d4..ab68d24e4b3 100644
--- a/compiler/rustc_attr/src/lib.rs
+++ b/compiler/rustc_attr/src/lib.rs
@@ -4,7 +4,7 @@
 //! The goal is to move the definition of `MetaItem` and things that don't need to be in `syntax`
 //! to this crate.
 
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 
 #[macro_use]
 extern crate rustc_macros;
diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs
index 36cd6c281b4..fd976b119b7 100644
--- a/compiler/rustc_builtin_macros/src/asm.rs
+++ b/compiler/rustc_builtin_macros/src/asm.rs
@@ -7,8 +7,10 @@ use rustc_errors::{Applicability, DiagnosticBuilder};
 use rustc_expand::base::{self, *};
 use rustc_parse::parser::Parser;
 use rustc_parse_format as parse;
+use rustc_session::lint;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::{InnerSpan, Span};
+use rustc_target::asm::InlineAsmArch;
 
 struct AsmArgs {
     templates: Vec<P<ast::Expr>>,
@@ -134,8 +136,8 @@ fn parse_args<'a>(
                 ast::InlineAsmOperand::InOut { reg, expr, late: true }
             }
         } else if p.eat_keyword(kw::Const) {
-            let expr = p.parse_expr()?;
-            ast::InlineAsmOperand::Const { expr }
+            let anon_const = p.parse_anon_const_expr()?;
+            ast::InlineAsmOperand::Const { anon_const }
         } else if p.eat_keyword(sym::sym) {
             let expr = p.parse_expr()?;
             match expr.kind {
@@ -424,6 +426,40 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
 
         let template_str = &template_str.as_str();
         let template_snippet = ecx.source_map().span_to_snippet(template_sp).ok();
+
+        if let Some(InlineAsmArch::X86 | InlineAsmArch::X86_64) = ecx.sess.asm_arch {
+            let find_span = |needle: &str| -> Span {
+                if let Some(snippet) = &template_snippet {
+                    if let Some(pos) = snippet.find(needle) {
+                        let end = pos
+                            + &snippet[pos..]
+                                .find(|c| matches!(c, '\n' | ';' | '\\' | '"'))
+                                .unwrap_or(snippet[pos..].len() - 1);
+                        let inner = InnerSpan::new(pos, end);
+                        return template_sp.from_inner(inner);
+                    }
+                }
+                template_sp
+            };
+
+            if template_str.contains(".intel_syntax") {
+                ecx.parse_sess().buffer_lint(
+                    lint::builtin::BAD_ASM_STYLE,
+                    find_span(".intel_syntax"),
+                    ecx.resolver.lint_node_id(ecx.current_expansion.id),
+                    "avoid using `.intel_syntax`, Intel syntax is the default",
+                );
+            }
+            if template_str.contains(".att_syntax") {
+                ecx.parse_sess().buffer_lint(
+                    lint::builtin::BAD_ASM_STYLE,
+                    find_span(".att_syntax"),
+                    ecx.resolver.lint_node_id(ecx.current_expansion.id),
+                    "avoid using `.att_syntax`, prefer using `options(att_syntax)` instead",
+                );
+            }
+        }
+
         let mut parser = parse::Parser::new(
             template_str,
             str_style,
diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs
new file mode 100644
index 00000000000..79dc857074d
--- /dev/null
+++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs
@@ -0,0 +1,269 @@
+use crate::util::check_builtin_macro_attribute;
+
+use rustc_ast as ast;
+use rustc_ast::mut_visit::MutVisitor;
+use rustc_ast::tokenstream::CanSynthesizeMissingTokens;
+use rustc_ast::visit::Visitor;
+use rustc_ast::{mut_visit, visit};
+use rustc_ast::{AstLike, Attribute};
+use rustc_expand::base::{Annotatable, ExtCtxt};
+use rustc_expand::config::StripUnconfigured;
+use rustc_expand::configure;
+use rustc_parse::parser::ForceCollect;
+use rustc_session::utils::FlattenNonterminals;
+
+use rustc_ast::ptr::P;
+use rustc_span::symbol::sym;
+use rustc_span::Span;
+use smallvec::SmallVec;
+
+crate fn expand(
+    ecx: &mut ExtCtxt<'_>,
+    _span: Span,
+    meta_item: &ast::MetaItem,
+    annotatable: Annotatable,
+) -> Vec<Annotatable> {
+    check_builtin_macro_attribute(ecx, meta_item, sym::cfg_eval);
+    cfg_eval(ecx, annotatable)
+}
+
+crate fn cfg_eval(ecx: &ExtCtxt<'_>, annotatable: Annotatable) -> Vec<Annotatable> {
+    let mut visitor = CfgEval {
+        cfg: &mut StripUnconfigured {
+            sess: ecx.sess,
+            features: ecx.ecfg.features,
+            config_tokens: true,
+        },
+    };
+    let annotatable = visitor.configure_annotatable(annotatable);
+    vec![annotatable]
+}
+
+struct CfgEval<'a, 'b> {
+    cfg: &'a mut StripUnconfigured<'b>,
+}
+
+fn flat_map_annotatable(vis: &mut impl MutVisitor, annotatable: Annotatable) -> Annotatable {
+    // Since the item itself has already been configured by the InvocationCollector,
+    // we know that fold result vector will contain exactly one element
+    match annotatable {
+        Annotatable::Item(item) => Annotatable::Item(vis.flat_map_item(item).pop().unwrap()),
+        Annotatable::TraitItem(item) => {
+            Annotatable::TraitItem(vis.flat_map_trait_item(item).pop().unwrap())
+        }
+        Annotatable::ImplItem(item) => {
+            Annotatable::ImplItem(vis.flat_map_impl_item(item).pop().unwrap())
+        }
+        Annotatable::ForeignItem(item) => {
+            Annotatable::ForeignItem(vis.flat_map_foreign_item(item).pop().unwrap())
+        }
+        Annotatable::Stmt(stmt) => {
+            Annotatable::Stmt(stmt.map(|stmt| vis.flat_map_stmt(stmt).pop().unwrap()))
+        }
+        Annotatable::Expr(mut expr) => Annotatable::Expr({
+            vis.visit_expr(&mut expr);
+            expr
+        }),
+        Annotatable::Arm(arm) => Annotatable::Arm(vis.flat_map_arm(arm).pop().unwrap()),
+        Annotatable::ExprField(field) => {
+            Annotatable::ExprField(vis.flat_map_expr_field(field).pop().unwrap())
+        }
+        Annotatable::PatField(fp) => {
+            Annotatable::PatField(vis.flat_map_pat_field(fp).pop().unwrap())
+        }
+        Annotatable::GenericParam(param) => {
+            Annotatable::GenericParam(vis.flat_map_generic_param(param).pop().unwrap())
+        }
+        Annotatable::Param(param) => Annotatable::Param(vis.flat_map_param(param).pop().unwrap()),
+        Annotatable::FieldDef(sf) => {
+            Annotatable::FieldDef(vis.flat_map_field_def(sf).pop().unwrap())
+        }
+        Annotatable::Variant(v) => Annotatable::Variant(vis.flat_map_variant(v).pop().unwrap()),
+    }
+}
+
+struct CfgFinder {
+    has_cfg_or_cfg_attr: bool,
+}
+
+impl CfgFinder {
+    fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
+        let mut finder = CfgFinder { has_cfg_or_cfg_attr: false };
+        match annotatable {
+            Annotatable::Item(item) => finder.visit_item(&item),
+            Annotatable::TraitItem(item) => finder.visit_assoc_item(&item, visit::AssocCtxt::Trait),
+            Annotatable::ImplItem(item) => finder.visit_assoc_item(&item, visit::AssocCtxt::Impl),
+            Annotatable::ForeignItem(item) => finder.visit_foreign_item(&item),
+            Annotatable::Stmt(stmt) => finder.visit_stmt(&stmt),
+            Annotatable::Expr(expr) => finder.visit_expr(&expr),
+            Annotatable::Arm(arm) => finder.visit_arm(&arm),
+            Annotatable::ExprField(field) => finder.visit_expr_field(&field),
+            Annotatable::PatField(field) => finder.visit_pat_field(&field),
+            Annotatable::GenericParam(param) => finder.visit_generic_param(&param),
+            Annotatable::Param(param) => finder.visit_param(&param),
+            Annotatable::FieldDef(field) => finder.visit_field_def(&field),
+            Annotatable::Variant(variant) => finder.visit_variant(&variant),
+        };
+        finder.has_cfg_or_cfg_attr
+    }
+}
+
+impl<'ast> visit::Visitor<'ast> for CfgFinder {
+    fn visit_attribute(&mut self, attr: &'ast Attribute) {
+        // We want short-circuiting behavior, so don't use the '|=' operator.
+        self.has_cfg_or_cfg_attr = self.has_cfg_or_cfg_attr
+            || attr
+                .ident()
+                .map_or(false, |ident| ident.name == sym::cfg || ident.name == sym::cfg_attr);
+    }
+}
+
+impl CfgEval<'_, '_> {
+    fn configure<T: AstLike>(&mut self, node: T) -> Option<T> {
+        self.cfg.configure(node)
+    }
+
+    pub fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Annotatable {
+        // Tokenizing and re-parsing the `Annotatable` can have a significant
+        // performance impact, so try to avoid it if possible
+        if !CfgFinder::has_cfg_or_cfg_attr(&annotatable) {
+            return annotatable;
+        }
+
+        // The majority of parsed attribute targets will never need to have early cfg-expansion
+        // run (e.g. they are not part of a `#[derive]` or `#[cfg_eval]` macro inoput).
+        // Therefore, we normally do not capture the necessary information about `#[cfg]`
+        // and `#[cfg_attr]` attributes during parsing.
+        //
+        // Therefore, when we actually *do* run early cfg-expansion, we need to tokenize
+        // and re-parse the attribute target, this time capturing information about
+        // the location of `#[cfg]` and `#[cfg_attr]` in the token stream. The tokenization
+        // process is lossless, so this process is invisible to proc-macros.
+
+        // FIXME - get rid of this clone
+        let nt = annotatable.clone().into_nonterminal();
+
+        let mut orig_tokens = rustc_parse::nt_to_tokenstream(
+            &nt,
+            &self.cfg.sess.parse_sess,
+            CanSynthesizeMissingTokens::No,
+        );
+
+        // 'Flatten' all nonterminals (i.e. `TokenKind::Interpolated`)
+        // to `None`-delimited groups containing the corresponding tokens. This
+        // is normally delayed until the proc-macro server actually needs to
+        // provide a `TokenKind::Interpolated` to a proc-macro. We do this earlier,
+        // so that we can handle cases like:
+        //
+        // ```rust
+        // #[cfg_eval] #[cfg] $item
+        //```
+        //
+        // where `$item` is `#[cfg_attr] struct Foo {}`. We want to make
+        // sure to evaluate *all* `#[cfg]` and `#[cfg_attr]` attributes - the simplest
+        // way to do this is to do a single parse of a stream without any nonterminals.
+        let mut flatten = FlattenNonterminals {
+            nt_to_tokenstream: rustc_parse::nt_to_tokenstream,
+            parse_sess: &self.cfg.sess.parse_sess,
+            synthesize_tokens: CanSynthesizeMissingTokens::No,
+        };
+        orig_tokens = flatten.process_token_stream(orig_tokens);
+
+        // Re-parse the tokens, setting the `capture_cfg` flag to save extra information
+        // to the captured `AttrAnnotatedTokenStream` (specifically, we capture
+        // `AttrAnnotatedTokenTree::AttributesData` for all occurences of `#[cfg]` and `#[cfg_attr]`)
+        let mut parser =
+            rustc_parse::stream_to_parser(&self.cfg.sess.parse_sess, orig_tokens, None);
+        parser.capture_cfg = true;
+        annotatable = match annotatable {
+            Annotatable::Item(_) => {
+                Annotatable::Item(parser.parse_item(ForceCollect::Yes).unwrap().unwrap())
+            }
+            Annotatable::TraitItem(_) => Annotatable::TraitItem(
+                parser.parse_trait_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
+            ),
+            Annotatable::ImplItem(_) => Annotatable::ImplItem(
+                parser.parse_impl_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
+            ),
+            Annotatable::ForeignItem(_) => Annotatable::ForeignItem(
+                parser.parse_foreign_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
+            ),
+            Annotatable::Stmt(_) => {
+                Annotatable::Stmt(P(parser.parse_stmt(ForceCollect::Yes).unwrap().unwrap()))
+            }
+            Annotatable::Expr(_) => Annotatable::Expr(parser.parse_expr_force_collect().unwrap()),
+            _ => unreachable!(),
+        };
+
+        // Now that we have our re-parsed `AttrAnnotatedTokenStream`, recursively configuring
+        // our attribute target will correctly the tokens as well.
+        flat_map_annotatable(self, annotatable)
+    }
+}
+
+impl MutVisitor for CfgEval<'_, '_> {
+    fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
+        self.cfg.configure_expr(expr);
+        mut_visit::noop_visit_expr(expr, self);
+    }
+
+    fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
+        let mut expr = configure!(self, expr);
+        mut_visit::noop_visit_expr(&mut expr, self);
+        Some(expr)
+    }
+
+    fn flat_map_generic_param(
+        &mut self,
+        param: ast::GenericParam,
+    ) -> SmallVec<[ast::GenericParam; 1]> {
+        mut_visit::noop_flat_map_generic_param(configure!(self, param), self)
+    }
+
+    fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
+        mut_visit::noop_flat_map_stmt(configure!(self, stmt), self)
+    }
+
+    fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
+        mut_visit::noop_flat_map_item(configure!(self, item), self)
+    }
+
+    fn flat_map_impl_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
+        mut_visit::noop_flat_map_assoc_item(configure!(self, item), self)
+    }
+
+    fn flat_map_trait_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
+        mut_visit::noop_flat_map_assoc_item(configure!(self, item), self)
+    }
+
+    fn flat_map_foreign_item(
+        &mut self,
+        foreign_item: P<ast::ForeignItem>,
+    ) -> SmallVec<[P<ast::ForeignItem>; 1]> {
+        mut_visit::noop_flat_map_foreign_item(configure!(self, foreign_item), self)
+    }
+
+    fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> {
+        mut_visit::noop_flat_map_arm(configure!(self, arm), self)
+    }
+
+    fn flat_map_expr_field(&mut self, field: ast::ExprField) -> SmallVec<[ast::ExprField; 1]> {
+        mut_visit::noop_flat_map_expr_field(configure!(self, field), self)
+    }
+
+    fn flat_map_pat_field(&mut self, fp: ast::PatField) -> SmallVec<[ast::PatField; 1]> {
+        mut_visit::noop_flat_map_pat_field(configure!(self, fp), self)
+    }
+
+    fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> {
+        mut_visit::noop_flat_map_param(configure!(self, p), self)
+    }
+
+    fn flat_map_field_def(&mut self, sf: ast::FieldDef) -> SmallVec<[ast::FieldDef; 1]> {
+        mut_visit::noop_flat_map_field_def(configure!(self, sf), self)
+    }
+
+    fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> {
+        mut_visit::noop_flat_map_variant(configure!(self, variant), self)
+    }
+}
diff --git a/compiler/rustc_builtin_macros/src/derive.rs b/compiler/rustc_builtin_macros/src/derive.rs
index fad64858ce3..1bb050a40ce 100644
--- a/compiler/rustc_builtin_macros/src/derive.rs
+++ b/compiler/rustc_builtin_macros/src/derive.rs
@@ -1,7 +1,8 @@
-use rustc_ast::{self as ast, token, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
+use crate::cfg_eval::cfg_eval;
+
+use rustc_ast::{self as ast, attr, token, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
 use rustc_errors::{struct_span_err, Applicability};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
-use rustc_expand::config::StripUnconfigured;
 use rustc_feature::AttributeTemplate;
 use rustc_parse::validate_attr;
 use rustc_session::Session;
@@ -25,52 +26,40 @@ impl MultiItemModifier for Expander {
             return ExpandResult::Ready(vec![item]);
         }
 
-        let template =
-            AttributeTemplate { list: Some("Trait1, Trait2, ..."), ..Default::default() };
-        let attr = ecx.attribute(meta_item.clone());
-        validate_attr::check_builtin_attribute(&sess.parse_sess, &attr, sym::derive, template);
+        let result =
+            ecx.resolver.resolve_derives(ecx.current_expansion.id, ecx.force_mode, &|| {
+                let template =
+                    AttributeTemplate { list: Some("Trait1, Trait2, ..."), ..Default::default() };
+                let attr = attr::mk_attr_outer(meta_item.clone());
+                validate_attr::check_builtin_attribute(
+                    &sess.parse_sess,
+                    &attr,
+                    sym::derive,
+                    template,
+                );
 
-        let derives: Vec<_> = attr
-            .meta_item_list()
-            .unwrap_or_default()
-            .into_iter()
-            .filter_map(|nested_meta| match nested_meta {
-                NestedMetaItem::MetaItem(meta) => Some(meta),
-                NestedMetaItem::Literal(lit) => {
-                    // Reject `#[derive("Debug")]`.
-                    report_unexpected_literal(sess, &lit);
-                    None
-                }
-            })
-            .map(|meta| {
-                // Reject `#[derive(Debug = "value", Debug(abc))]`, but recover the paths.
-                report_path_args(sess, &meta);
-                meta.path
-            })
-            .collect();
+                attr.meta_item_list()
+                    .unwrap_or_default()
+                    .into_iter()
+                    .filter_map(|nested_meta| match nested_meta {
+                        NestedMetaItem::MetaItem(meta) => Some(meta),
+                        NestedMetaItem::Literal(lit) => {
+                            // Reject `#[derive("Debug")]`.
+                            report_unexpected_literal(sess, &lit);
+                            None
+                        }
+                    })
+                    .map(|meta| {
+                        // Reject `#[derive(Debug = "value", Debug(abc))]`, but recover the paths.
+                        report_path_args(sess, &meta);
+                        meta.path
+                    })
+                    .map(|path| (path, None))
+                    .collect()
+            });
 
-        // FIXME: Try to cache intermediate results to avoid collecting same paths multiple times.
-        match ecx.resolver.resolve_derives(ecx.current_expansion.id, derives, ecx.force_mode) {
-            Ok(()) => {
-                let mut visitor =
-                    StripUnconfigured { sess, features: ecx.ecfg.features, modified: false };
-                let mut item = visitor.fully_configure(item);
-                if visitor.modified {
-                    // Erase the tokens if cfg-stripping modified the item
-                    // This will cause us to synthesize fake tokens
-                    // when `nt_to_tokenstream` is called on this item.
-                    match &mut item {
-                        Annotatable::Item(item) => item,
-                        Annotatable::Stmt(stmt) => match &mut stmt.kind {
-                            StmtKind::Item(item) => item,
-                            _ => unreachable!(),
-                        },
-                        _ => unreachable!(),
-                    }
-                    .tokens = None;
-                }
-                ExpandResult::Ready(vec![item])
-            }
+        match result {
+            Ok(()) => ExpandResult::Ready(cfg_eval(ecx, item)),
             Err(Indeterminate) => ExpandResult::Retry(item),
         }
     }
diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
index d498c8e1727..a3decff3ae7 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
@@ -541,7 +541,7 @@ impl<'a> TraitDef<'a> {
             self.generics.to_generics(cx, self.span, type_ident, generics);
 
         // Create the generic parameters
-        params.extend(generics.params.iter().map(|param| match param.kind {
+        params.extend(generics.params.iter().map(|param| match &param.kind {
             GenericParamKind::Lifetime { .. } => param.clone(),
             GenericParamKind::Type { .. } => {
                 // I don't think this can be moved out of the loop, since
@@ -561,7 +561,18 @@ impl<'a> TraitDef<'a> {
 
                 cx.typaram(self.span, param.ident, vec![], bounds, None)
             }
-            GenericParamKind::Const { .. } => param.clone(),
+            GenericParamKind::Const { ty, kw_span, .. } => {
+                let const_nodefault_kind = GenericParamKind::Const {
+                    ty: ty.clone(),
+                    kw_span: kw_span.clone(),
+
+                    // We can't have default values inside impl block
+                    default: None,
+                };
+                let mut param_clone = param.clone();
+                param_clone.kind = const_nodefault_kind;
+                param_clone
+            }
         }));
 
         // and similarly for where clauses
@@ -1034,7 +1045,7 @@ impl<'a> MethodDef<'a> {
         // make a series of nested matches, to destructure the
         // structs. This is actually right-to-left, but it shouldn't
         // matter.
-        for (arg_expr, pat) in self_args.iter().zip(patterns) {
+        for (arg_expr, pat) in iter::zip(self_args, patterns) {
             body = cx.expr_match(
                 trait_.span,
                 arg_expr.clone(),
@@ -1351,7 +1362,7 @@ impl<'a> MethodDef<'a> {
             let mut discriminant_test = cx.expr_bool(sp, true);
 
             let mut first_ident = None;
-            for (&ident, self_arg) in vi_idents.iter().zip(&self_args) {
+            for (&ident, self_arg) in iter::zip(&vi_idents, &self_args) {
                 let self_addr = cx.expr_addr_of(sp, self_arg.clone());
                 let variant_value =
                     deriving::call_intrinsic(cx, sp, sym::discriminant_value, vec![self_addr]);
@@ -1571,14 +1582,12 @@ impl<'a> TraitDef<'a> {
         let subpats = self.create_subpatterns(cx, paths, mutbl, use_temporaries);
         let pattern = match *struct_def {
             VariantData::Struct(..) => {
-                let field_pats = subpats
-                    .into_iter()
-                    .zip(&ident_exprs)
+                let field_pats = iter::zip(subpats, &ident_exprs)
                     .map(|(pat, &(sp, ident, ..))| {
                         if ident.is_none() {
                             cx.span_bug(sp, "a braced struct with unnamed fields in `derive`");
                         }
-                        ast::FieldPat {
+                        ast::PatField {
                             ident: ident.unwrap(),
                             is_shorthand: false,
                             attrs: ast::AttrVec::new(),
diff --git a/compiler/rustc_builtin_macros/src/global_allocator.rs b/compiler/rustc_builtin_macros/src/global_allocator.rs
index 9b43c11f0f3..a97cac7e514 100644
--- a/compiler/rustc_builtin_macros/src/global_allocator.rs
+++ b/compiler/rustc_builtin_macros/src/global_allocator.rs
@@ -14,31 +14,31 @@ pub fn expand(
     ecx: &mut ExtCtxt<'_>,
     _span: Span,
     meta_item: &ast::MetaItem,
-    mut item: Annotatable,
+    item: Annotatable,
 ) -> Vec<Annotatable> {
     check_builtin_macro_attribute(ecx, meta_item, sym::global_allocator);
 
-    let not_static = |item: Annotatable| {
+    let orig_item = item.clone();
+    let not_static = || {
         ecx.sess.parse_sess.span_diagnostic.span_err(item.span(), "allocators must be statics");
-        vec![item]
+        vec![orig_item.clone()]
     };
-    let orig_item = item.clone();
-    let mut is_stmt = false;
 
     // Allow using `#[global_allocator]` on an item statement
-    if let Annotatable::Stmt(stmt) = &item {
-        if let StmtKind::Item(item_) = &stmt.kind {
-            item = Annotatable::Item(item_.clone());
-            is_stmt = true;
-        }
-    }
-
-    let item = match item {
+    // FIXME - if we get deref patterns, use them to reduce duplication here
+    let (item, is_stmt) = match &item {
         Annotatable::Item(item) => match item.kind {
-            ItemKind::Static(..) => item,
-            _ => return not_static(Annotatable::Item(item)),
+            ItemKind::Static(..) => (item, false),
+            _ => return not_static(),
+        },
+        Annotatable::Stmt(stmt) => match &stmt.kind {
+            StmtKind::Item(item_) => match item_.kind {
+                ItemKind::Static(..) => (item_, true),
+                _ => return not_static(),
+            },
+            _ => return not_static(),
         },
-        _ => return not_static(item),
+        _ => return not_static(),
     };
 
     // Generate a bunch of new items using the AllocFnFactory
diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs
index 9a3c914337c..d7926ed0e0b 100644
--- a/compiler/rustc_builtin_macros/src/lib.rs
+++ b/compiler/rustc_builtin_macros/src/lib.rs
@@ -7,10 +7,12 @@
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_quote)]
+#![recursion_limit = "256"]
 
 extern crate proc_macro;
 
@@ -24,6 +26,7 @@ mod asm;
 mod assert;
 mod cfg;
 mod cfg_accessible;
+mod cfg_eval;
 mod compile_error;
 mod concat;
 mod concat_idents;
@@ -89,6 +92,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
     register_attr! {
         bench: test::expand_bench,
         cfg_accessible: cfg_accessible::Expander,
+        cfg_eval: cfg_eval::expand,
         derive: derive::Expander,
         global_allocator: global_allocator::expand,
         test: test::expand_test,
diff --git a/compiler/rustc_builtin_macros/src/source_util.rs b/compiler/rustc_builtin_macros/src/source_util.rs
index 28efd483c86..4aafcb2fb6d 100644
--- a/compiler/rustc_builtin_macros/src/source_util.rs
+++ b/compiler/rustc_builtin_macros/src/source_util.rs
@@ -4,7 +4,7 @@ use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{self, *};
-use rustc_expand::module::DirectoryOwnership;
+use rustc_expand::module::DirOwnership;
 use rustc_parse::parser::{ForceCollect, Parser};
 use rustc_parse::{self, new_parser_from_file};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
@@ -101,7 +101,7 @@ pub fn expand_include<'cx>(
         None => return DummyResult::any(sp),
     };
     // The file will be added to the code map by the parser
-    let mut file = match cx.resolve_path(file, sp) {
+    let file = match cx.resolve_path(file, sp) {
         Ok(f) => f,
         Err(mut err) => {
             err.emit();
@@ -114,10 +114,9 @@ pub fn expand_include<'cx>(
     // then the path of `bar.rs` should be relative to the directory of `file`.
     // See https://github.com/rust-lang/rust/pull/69838/files#r395217057 for a discussion.
     // `MacroExpander::fully_expand_fragment` later restores, so "stack discipline" is maintained.
-    file.pop();
-    cx.current_expansion.directory_ownership = DirectoryOwnership::Owned { relative: None };
-    let mod_path = cx.current_expansion.module.mod_path.clone();
-    cx.current_expansion.module = Rc::new(ModuleData { mod_path, directory: file });
+    let dir_path = file.parent().unwrap_or(&file).to_owned();
+    cx.current_expansion.module = Rc::new(cx.current_expansion.module.with_dir_path(dir_path));
+    cx.current_expansion.dir_ownership = DirOwnership::Owned { relative: None };
 
     struct ExpandResult<'a> {
         p: Parser<'a>,
diff --git a/compiler/rustc_builtin_macros/src/standard_library_imports.rs b/compiler/rustc_builtin_macros/src/standard_library_imports.rs
index f446e6be84c..fbd8be22a9d 100644
--- a/compiler/rustc_builtin_macros/src/standard_library_imports.rs
+++ b/compiler/rustc_builtin_macros/src/standard_library_imports.rs
@@ -2,7 +2,7 @@ use rustc_ast as ast;
 use rustc_expand::base::{ExtCtxt, ResolverExpand};
 use rustc_expand::expand::ExpansionConfig;
 use rustc_session::Session;
-use rustc_span::edition::Edition;
+use rustc_span::edition::Edition::*;
 use rustc_span::hygiene::AstPass;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::DUMMY_SP;
@@ -13,7 +13,7 @@ pub fn inject(
     sess: &Session,
     alt_std_name: Option<Symbol>,
 ) -> ast::Crate {
-    let rust_2018 = sess.parse_sess.edition >= Edition::Edition2018;
+    let edition = sess.parse_sess.edition;
 
     // the first name in this list is the crate name of the crate with the prelude
     let names: &[Symbol] = if sess.contains_name(&krate.attrs, sym::no_core) {
@@ -42,7 +42,11 @@ pub fn inject(
 
     // .rev() to preserve ordering above in combination with insert(0, ...)
     for &name in names.iter().rev() {
-        let ident = if rust_2018 { Ident::new(name, span) } else { Ident::new(name, call_site) };
+        let ident = if edition >= Edition2018 {
+            Ident::new(name, span)
+        } else {
+            Ident::new(name, call_site)
+        };
         krate.items.insert(
             0,
             cx.item(
@@ -58,14 +62,18 @@ pub fn inject(
     // the one with the prelude.
     let name = names[0];
 
-    let import_path = if rust_2018 {
-        [name, sym::prelude, sym::v1].iter().map(|symbol| Ident::new(*symbol, span)).collect()
-    } else {
-        [kw::PathRoot, name, sym::prelude, sym::v1]
-            .iter()
-            .map(|symbol| Ident::new(*symbol, span))
-            .collect()
-    };
+    let root = (edition == Edition2015).then(|| kw::PathRoot);
+
+    let import_path = root
+        .iter()
+        .chain(&[name, sym::prelude])
+        .chain(&[match edition {
+            Edition2015 => sym::rust_2015,
+            Edition2018 => sym::rust_2018,
+            Edition2021 => sym::rust_2021,
+        }])
+        .map(|&symbol| Ident::new(symbol, span))
+        .collect();
 
     let use_item = cx.item(
         span,
diff --git a/compiler/rustc_builtin_macros/src/test_harness.rs b/compiler/rustc_builtin_macros/src/test_harness.rs
index 28e82597843..c8a7ff67b4d 100644
--- a/compiler/rustc_builtin_macros/src/test_harness.rs
+++ b/compiler/rustc_builtin_macros/src/test_harness.rs
@@ -142,7 +142,7 @@ fn entry_point_type(sess: &Session, item: &ast::Item, depth: usize) -> EntryPoin
         ast::ItemKind::Fn(..) => {
             if sess.contains_name(&item.attrs, sym::start) {
                 EntryPointType::Start
-            } else if sess.contains_name(&item.attrs, sym::main) {
+            } else if sess.contains_name(&item.attrs, sym::rustc_main) {
                 EntryPointType::MainAttr
             } else if item.ident.name == sym::main {
                 if depth == 1 {
@@ -187,7 +187,7 @@ impl<'a> MutVisitor for EntryPointCleaner<'a> {
                     let attrs = attrs
                         .into_iter()
                         .filter(|attr| {
-                            !self.sess.check_name(attr, sym::main)
+                            !self.sess.check_name(attr, sym::rustc_main)
                                 && !self.sess.check_name(attr, sym::start)
                         })
                         .chain(iter::once(allow_dead_code))
@@ -220,7 +220,7 @@ fn generate_test_harness(
     let expn_id = ext_cx.resolver.expansion_for_ast_pass(
         DUMMY_SP,
         AstPass::TestHarness,
-        &[sym::main, sym::test, sym::rustc_attrs],
+        &[sym::test, sym::rustc_attrs],
         None,
     );
     let def_site = DUMMY_SP.with_def_site_ctxt(expn_id);
@@ -247,7 +247,7 @@ fn generate_test_harness(
 /// By default this expands to
 ///
 /// ```
-/// #[main]
+/// #[rustc_main]
 /// pub fn main() {
 ///     extern crate test;
 ///     test::test_main_static(&[
@@ -297,8 +297,8 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
     let test_extern_stmt =
         ecx.stmt_item(sp, ecx.item(sp, test_id, vec![], ast::ItemKind::ExternCrate(None)));
 
-    // #[main]
-    let main_meta = ecx.meta_word(sp, sym::main);
+    // #[rustc_main]
+    let main_meta = ecx.meta_word(sp, sym::rustc_main);
     let main_attr = ecx.attribute(main_meta);
 
     // pub fn main() { ... }
diff --git a/compiler/rustc_codegen_cranelift/.cirrus.yml b/compiler/rustc_codegen_cranelift/.cirrus.yml
new file mode 100644
index 00000000000..e173df423a7
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/.cirrus.yml
@@ -0,0 +1,25 @@
+task:
+  name: freebsd
+  freebsd_instance:
+    image: freebsd-12-1-release-amd64
+  setup_rust_script:
+    - pkg install -y curl git bash
+    - curl https://sh.rustup.rs -sSf --output rustup.sh
+    - sh rustup.sh --default-toolchain none -y --profile=minimal
+  cargo_bin_cache:
+    folder: ~/.cargo/bin
+  target_cache:
+    folder: target
+  prepare_script:
+    - . $HOME/.cargo/env
+    - git config --global user.email "user@example.com"
+    - git config --global user.name "User"
+    - ./prepare.sh
+  test_script:
+    - . $HOME/.cargo/env
+    - # Enable backtraces for easier debugging
+    - export RUST_BACKTRACE=1
+    - # Reduce amount of benchmark runs as they are slow
+    - export COMPILE_RUNS=2
+    - export RUN_RUNS=2
+    - ./test.sh
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/bootstrap_rustc.yml b/compiler/rustc_codegen_cranelift/.github/workflows/bootstrap_rustc.yml
deleted file mode 100644
index 8c94a0aa5e6..00000000000
--- a/compiler/rustc_codegen_cranelift/.github/workflows/bootstrap_rustc.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-name: Bootstrap rustc using cg_clif
-
-on:
-  - push
-
-jobs:
-  bootstrap_rustc:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Cache cargo installed crates
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v2
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
-    - name: Cache cargo target dir
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
-
-    - name: Prepare dependencies
-      run: |
-        git config --global user.email "user@example.com"
-        git config --global user.name "User"
-        ./prepare.sh
-
-    - name: Test
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        ./scripts/test_bootstrap.sh
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
index 20c58423a0c..2ac516381cf 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
@@ -7,14 +7,18 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
 
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        env:
-          - BACKEND: ""
-          - BACKEND: --oldbe
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+          # cross-compile from Linux to Windows using mingw
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
 
     steps:
     - uses: actions/checkout@v2
@@ -39,6 +43,12 @@ jobs:
         path: target
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
+    - name: Install MinGW toolchain and wine
+      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: |
+        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+        rustup target add x86_64-pc-windows-gnu
+
     - name: Prepare dependencies
       run: |
         git config --global user.email "user@example.com"
@@ -46,6 +56,8 @@ jobs:
         ./prepare.sh
 
     - name: Test
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
       run: |
         # Enable backtraces for easier debugging
         export RUST_BACKTRACE=1
@@ -54,12 +66,16 @@ jobs:
         export COMPILE_RUNS=2
         export RUN_RUNS=2
 
-        ./test.sh $BACKEND
+        # Enable extra checks
+        export CG_CLIF_ENABLE_VERIFIER=1
+
+        ./test.sh
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz build
 
     - name: Upload prebuilt cg_clif
+      if: matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
       uses: actions/upload-artifact@v2
       with:
         name: cg_clif-${{ runner.os }}
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
new file mode 100644
index 00000000000..e01a92598ba
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
@@ -0,0 +1,82 @@
+name: Various rustc tests
+
+on:
+  - push
+
+jobs:
+  bootstrap_rustc:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_bootstrap.sh
+  rustc_test_suite:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_rustc_tests.sh
diff --git a/compiler/rustc_codegen_cranelift/.vscode/settings.json b/compiler/rustc_codegen_cranelift/.vscode/settings.json
index 19ea41563df..0cd576e160f 100644
--- a/compiler/rustc_codegen_cranelift/.vscode/settings.json
+++ b/compiler/rustc_codegen_cranelift/.vscode/settings.json
@@ -1,8 +1,8 @@
 {
     // source for rustc_* is not included in the rust-src component; disable the errors about this
-    "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"],
+    "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"],
     "rust-analyzer.assist.importMergeBehavior": "last",
-    "rust-analyzer.cargo.loadOutDirsFromCheck": true,
+    "rust-analyzer.cargo.runBuildScripts": true,
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml",
         //"./build_sysroot/sysroot_src/src/libstd/Cargo.toml",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index 5495cfa5eaa..dc1cd336e15 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "anyhow"
 version = "1.0.38"
@@ -30,18 +32,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
 
 [[package]]
-name = "cc"
-version = "1.0.66"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
-
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
-
-[[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -49,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "byteorder",
  "cranelift-bforest",
@@ -75,8 +65,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -84,18 +74,18 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -105,8 +95,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -123,8 +113,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -135,18 +125,17 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen",
- "raw-cpuid",
  "target-lexicon",
 ]
 
 [[package]]
 name = "cranelift-object"
-version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -162,7 +151,7 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -219,9 +208,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.82"
+version = "0.2.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
+checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
 
 [[package]]
 name = "libloading"
@@ -229,17 +218,17 @@ version = "0.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "winapi",
 ]
 
 [[package]]
 name = "log"
-version = "0.4.13"
+version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if",
 ]
 
 [[package]]
@@ -253,9 +242,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.22.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397"
+checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
 dependencies = [
  "crc32fast",
  "indexmap",
@@ -272,25 +261,14 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
 dependencies = [
  "proc-macro2",
 ]
 
 [[package]]
-name = "raw-cpuid"
-version = "8.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73"
-dependencies = [
- "bitflags",
- "cc",
- "rustc_version",
-]
-
-[[package]]
 name = "regalloc"
 version = "0.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -338,30 +316,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-dependencies = [
- "semver",
-]
-
-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-dependencies = [
- "semver-parser",
-]
-
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-
-[[package]]
 name = "smallvec"
 version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -369,9 +323,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
 [[package]]
 name = "syn"
-version = "1.0.58"
+version = "1.0.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5"
+checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -380,24 +334,24 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.11.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9"
+checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95"
 
 [[package]]
 name = "thiserror"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
+checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
+checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 3820fce6d1e..60946ab2808 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -9,14 +9,14 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x86", "x64"] }
+cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x64"] }
 cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
-object = { version = "0.22.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
+object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
@@ -38,7 +38,6 @@ smallvec = "1.6.1"
 default = ["jit", "inline_asm"]
 jit = ["cranelift-jit", "libloading"]
 inline_asm = []
-oldbe = []
 
 [profile.dev]
 # By compiling dependencies with optimizations, performing tests gets much faster.
@@ -76,3 +75,6 @@ debug = false
 [profile.release.package.syn]
 opt-level = 0
 debug = false
+
+[package.metadata.rust-analyzer]
+rustc_private = true
diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md
index 6fa5eebdc2f..ffe1d9a1e65 100644
--- a/compiler/rustc_codegen_cranelift/Readme.md
+++ b/compiler/rustc_codegen_cranelift/Readme.md
@@ -34,70 +34,19 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo bui
 
 Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
 
-### Cargo
-
 In the directory with your project (where you can do the usual `cargo build`), run:
 
 ```bash
-$ $cg_clif_dir/build/cargo.sh run
-```
-
-This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend.
-
-### Rustc
-
-> You should prefer using the Cargo method.
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
-```
-
-### Jit mode
-
-In jit mode cg_clif will immediately execute your code without creating an executable file.
-
-> This requires all dependencies to be available as dynamic library.
-> The jit mode will probably need cargo integration to make this possible.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh jit
-```
-
-or
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
-```
-
-There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
-first called. It currently does not work with multi-threaded programs. When a not yet compiled
-function is called from another thread than the main thread, you will get an ICE.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh lazy-jit
+$ $cg_clif_dir/build/cargo.sh build
 ```
 
-### Shell
-
-These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
-
-```bash
-function jit_naked() {
-    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
-}
-
-function jit() {
-    jit_naked "fn main() { $@ }"
-}
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
 
-function jit_calc() {
-    jit 'println!("0x{:x}", ' $@ ');';
-}
-```
+For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md).
 
 ## Env vars
 
-[see env_vars.md](docs/env_vars.md)
+See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift.
 
 ## Not yet supported
 
@@ -106,3 +55,20 @@ function jit_calc() {
       `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You
       have to specify specific registers instead.
 * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work)
+
+## License
+
+Licensed under either of
+
+  * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or
+    http://www.apache.org/licenses/LICENSE-2.0)
+  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+    http://opensource.org/licenses/MIT)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in the work by you shall be dual licensed as above, without any
+additional terms or conditions.
diff --git a/compiler/rustc_codegen_cranelift/build.sh b/compiler/rustc_codegen_cranelift/build.sh
index 598ce35ecea..76bc1884334 100755
--- a/compiler/rustc_codegen_cranelift/build.sh
+++ b/compiler/rustc_codegen_cranelift/build.sh
@@ -1,11 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 # Settings
 export CHANNEL="release"
 build_sysroot="clif"
 target_dir='build'
-oldbe=''
 while [[ $# != 0 ]]; do
     case $1 in
         "--debug")
@@ -19,12 +18,9 @@ while [[ $# != 0 ]]; do
             target_dir=$2
             shift
             ;;
-        "--oldbe")
-            oldbe='--features oldbe'
-            ;;
         *)
             echo "Unknown flag '$1'"
-            echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--oldbe]"
+            echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]"
             exit 1
             ;;
     esac
@@ -34,19 +30,19 @@ done
 # Build cg_clif
 unset CARGO_TARGET_DIR
 unamestr=$(uname)
-if [[ "$unamestr" == 'Linux' ]]; then
+if [[ "$unamestr" == 'Linux' || "$unamestr" == "FreeBSD" ]]; then
    export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS
 elif [[ "$unamestr" == 'Darwin' ]]; then
    export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS
    dylib_ext='dylib'
 else
-   echo "Unsupported os"
+   echo "Unsupported os $unamestr"
    exit 1
 fi
 if [[ "$CHANNEL" == "release" ]]; then
-    cargo build $oldbe --release
+    cargo build --release
 else
-    cargo build $oldbe
+    cargo build
 fi
 
 source scripts/ext_config.sh
@@ -59,6 +55,7 @@ ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib
 ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir"
 
 mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
+mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/"
 if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
     cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
 fi
@@ -68,12 +65,18 @@ case "$build_sysroot" in
         ;;
     "llvm")
         cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/"
+        fi
         ;;
     "clif")
         echo "[BUILD] sysroot"
         dir=$(pwd)
         cd "$target_dir"
         time "$dir/build_sysroot/build_sysroot.sh"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh"
+        fi
         cp lib/rustlib/*/lib/libstd-* lib/
         ;;
     *)
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
index 0da9999c172..09c5d7590ab 100644
--- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "addr2line"
 version = "0.14.1"
@@ -14,9 +16,9 @@ dependencies = [
 
 [[package]]
 name = "adler"
-version = "0.2.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -31,15 +33,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "alloc_system"
-version = "0.0.0"
-dependencies = [
- "compiler_builtins",
- "core",
- "libc",
-]
-
-[[package]]
 name = "autocfg"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -47,9 +40,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "cc"
-version = "1.0.66"
+version = "1.0.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
+checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
 
 [[package]]
 name = "cfg-if"
@@ -117,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.9.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -139,18 +132,18 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.84"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
+checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
 dependencies = [
  "rustc-std-workspace-core",
 ]
 
 [[package]]
 name = "miniz_oxide"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
 dependencies = [
  "adler",
  "autocfg",
@@ -258,7 +251,6 @@ name = "sysroot"
 version = "0.0.0"
 dependencies = [
  "alloc",
- "alloc_system",
  "compiler_builtins",
  "core",
  "std",
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml
index 82516c98af2..04748d5dbab 100644
--- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml
@@ -9,8 +9,6 @@ alloc = { path = "./sysroot_src/library/alloc" }
 std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
 test = { path = "./sysroot_src/library/test" }
 
-alloc_system = { path = "./alloc_system" }
-
 compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] }
 
 [patch.crates-io]
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml b/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml
deleted file mode 100644
index 9fffca84300..00000000000
--- a/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml
+++ /dev/null
@@ -1,13 +0,0 @@
-[package]
-authors = ["The Rust Project Developers", "bjorn3 (edited to be usable outside the rust source)"]
-name = "alloc_system"
-version = "0.0.0"
-[lib]
-name = "alloc_system"
-path = "lib.rs"
-test = false
-doc = false
-[dependencies]
-core = { path = "../sysroot_src/library/core" }
-libc = { version = "0.2.43", features = ['rustc-dep-of-std'], default-features = false }
-compiler_builtins = "0.1"
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh b/compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh
index 282ce4a582c..0354304e55b 100755
--- a/compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Requires the CHANNEL env var to be set to `debug` or `release.`
 
@@ -28,7 +28,7 @@ export __CARGO_DEFAULT_LIB_METADATA="cg_clif"
 if [[ "$1" != "--debug" ]]; then
     sysroot_channel='release'
     # FIXME Enable incremental again once rust-lang/rust#74946 is fixed
-    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=2" cargo build --target "$TARGET_TRIPLE" --release
+    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target "$TARGET_TRIPLE" --release
 else
     sysroot_channel='debug'
     cargo build --target "$TARGET_TRIPLE"
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh b/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh
index d3b87e02ba8..c90205db0fb 100755
--- a/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 cd "$(dirname "$0")"
 
@@ -33,7 +33,7 @@ git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/
 pushd compiler-builtins
 git checkout -- .
 git checkout 0.1.39
-git apply ../../crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch
+git apply ../../crate_patches/000*-compiler-builtins-*.patch
 popd
 
 echo "Successfully prepared sysroot source for building"
diff --git a/compiler/rustc_codegen_cranelift/clean_all.sh b/compiler/rustc_codegen_cranelift/clean_all.sh
index b47efe72bce..a7bbeb05cac 100755
--- a/compiler/rustc_codegen_cranelift/clean_all.sh
+++ b/compiler/rustc_codegen_cranelift/clean_all.sh
@@ -1,4 +1,4 @@
-#!/bin/bash --verbose
+#!/usr/bin/env bash
 set -e
 
 rm -rf target/ build/ build_sysroot/{sysroot_src/,target/,compiler-builtins/} perf.data{,.old}
diff --git a/compiler/rustc_codegen_cranelift/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch
new file mode 100644
index 00000000000..7daea99f579
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch
@@ -0,0 +1,48 @@
+From 1d574bf5e32d51641dcacaf8ef777e95b44f6f2a Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Thu, 18 Feb 2021 18:30:55 +0100
+Subject: [PATCH] Disable 128bit atomic operations
+
+Cranelift doesn't support them yet
+---
+ src/mem/mod.rs | 12 ------------
+ 1 file changed, 12 deletions(-)
+
+diff --git a/src/mem/mod.rs b/src/mem/mod.rs
+index 107762c..2d1ae10 100644
+--- a/src/mem/mod.rs
++++ b/src/mem/mod.rs
+@@ -137,10 +137,6 @@ intrinsics! {
+     pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+         memcpy_element_unordered_atomic(dest, src, bytes);
+     }
+-    #[cfg(target_has_atomic_load_store = "128")]
+-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+-        memcpy_element_unordered_atomic(dest, src, bytes);
+-    }
+ 
+     #[cfg(target_has_atomic_load_store = "8")]
+     pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
+@@ -158,10 +154,6 @@ intrinsics! {
+     pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+         memmove_element_unordered_atomic(dest, src, bytes);
+     }
+-    #[cfg(target_has_atomic_load_store = "128")]
+-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+-        memmove_element_unordered_atomic(dest, src, bytes);
+-    }
+ 
+     #[cfg(target_has_atomic_load_store = "8")]
+     pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
+@@ -179,8 +171,4 @@ intrinsics! {
+     pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
+         memset_element_unordered_atomic(s, c, bytes);
+     }
+-    #[cfg(target_has_atomic_load_store = "128")]
+-    pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
+-        memset_element_unordered_atomic(s, c, bytes);
+-    }
+ }
+-- 
+2.26.2.7.g19db9cfb68
+
diff --git a/compiler/rustc_codegen_cranelift/docs/env_vars.md b/compiler/rustc_codegen_cranelift/docs/env_vars.md
index f0a0a6ad42e..f7fde1b4f3a 100644
--- a/compiler/rustc_codegen_cranelift/docs/env_vars.md
+++ b/compiler/rustc_codegen_cranelift/docs/env_vars.md
@@ -8,5 +8,8 @@
     to make it possible to use incremental mode for all analyses performed by rustc without caching
     object files when their content should have been changed by a change to cg_clif.</dd>
     <dt>CG_CLIF_DISPLAY_CG_TIME</dt>
-    <dd>If "1", display the time it took to perform codegen for a crate</dd>
+    <dd>If "1", display the time it took to perform codegen for a crate.</dd>
+    <dt>CG_CLIF_ENABLE_VERIFIER</dt>
+    <dd>Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once
+    before passing the clif ir to Cranelift for compilation.</dt>
 </dl>
diff --git a/compiler/rustc_codegen_cranelift/docs/usage.md b/compiler/rustc_codegen_cranelift/docs/usage.md
new file mode 100644
index 00000000000..3eee3b554e3
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/docs/usage.md
@@ -0,0 +1,66 @@
+# Usage
+
+rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
+
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
+
+## Cargo
+
+In the directory with your project (where you can do the usual `cargo build`), run:
+
+```bash
+$ $cg_clif_dir/build/cargo.sh build
+```
+
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+
+## Rustc
+
+> You should prefer using the Cargo method.
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
+```
+
+## Jit mode
+
+In jit mode cg_clif will immediately execute your code without creating an executable file.
+
+> This requires all dependencies to be available as dynamic library.
+> The jit mode will probably need cargo integration to make this possible.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh jit
+```
+
+or
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
+```
+
+There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
+first called. It currently does not work with multi-threaded programs. When a not yet compiled
+function is called from another thread than the main thread, you will get an ICE.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh lazy-jit
+```
+
+## Shell
+
+These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
+
+```bash
+function jit_naked() {
+    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
+}
+
+function jit() {
+    jit_naked "fn main() { $@ }"
+}
+
+function jit_calc() {
+    jit 'println!("0x{:x}", ' $@ ');';
+}
+```
diff --git a/compiler/rustc_codegen_cranelift/example/alloc_example.rs b/compiler/rustc_codegen_cranelift/example/alloc_example.rs
index f59600ebb33..71e93e87b6c 100644
--- a/compiler/rustc_codegen_cranelift/example/alloc_example.rs
+++ b/compiler/rustc_codegen_cranelift/example/alloc_example.rs
@@ -1,4 +1,4 @@
-#![feature(start, box_syntax, alloc_system, core_intrinsics, alloc_prelude, alloc_error_handler)]
+#![feature(start, box_syntax, core_intrinsics, alloc_prelude, alloc_error_handler)]
 #![no_std]
 
 extern crate alloc;
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/lib.rs b/compiler/rustc_codegen_cranelift/example/alloc_system.rs
index c832d5e5ebb..5f66ca67f2d 100644
--- a/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/lib.rs
+++ b/compiler/rustc_codegen_cranelift/example/alloc_system.rs
@@ -8,66 +8,24 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 #![no_std]
-#![allow(unused_attributes)]
-#![unstable(feature = "alloc_system",
-            reason = "this library is unlikely to be stabilized in its current \
-                      form or name",
-            issue = "32838")]
-#![feature(allocator_api)]
-#![feature(core_intrinsics)]
-#![feature(nll)]
-#![feature(staged_api)]
-#![feature(rustc_attrs)]
-#![feature(alloc_layout_extra)]
-#![cfg_attr(
-    all(target_arch = "wasm32", not(target_os = "emscripten")),
-    feature(integer_atomics, stdsimd)
-)]
+#![feature(allocator_api, rustc_private)]
 #![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
+
 // The minimum alignment guaranteed by the architecture. This value is used to
 // add fast paths for low alignment values.
 #[cfg(all(any(target_arch = "x86",
               target_arch = "arm",
               target_arch = "mips",
               target_arch = "powerpc",
-              target_arch = "powerpc64",
-              target_arch = "asmjs",
-              target_arch = "wasm32")))]
-#[allow(dead_code)]
+              target_arch = "powerpc64")))]
 const MIN_ALIGN: usize = 8;
 #[cfg(all(any(target_arch = "x86_64",
               target_arch = "aarch64",
               target_arch = "mips64",
               target_arch = "s390x",
               target_arch = "sparc64")))]
-#[allow(dead_code)]
 const MIN_ALIGN: usize = 16;
 
-/// The default memory allocator provided by the operating system.
-///
-/// This is based on `malloc` on Unix platforms and `HeapAlloc` on Windows,
-/// plus related functions.
-///
-/// This type can be used in a `static` item
-/// with the `#[global_allocator]` attribute
-/// to force the global allocator to be the system’s one.
-/// (The default is jemalloc for executables, on some platforms.)
-///
-/// ```rust
-/// use std::alloc::System;
-///
-/// #[global_allocator]
-/// static A: System = System;
-///
-/// fn main() {
-///     let a = Box::new(4); // Allocates from the system allocator.
-///     println!("{}", a);
-/// }
-/// ```
-///
-/// It can also be used directly to allocate memory
-/// independently of the standard library’s global allocator.
-#[stable(feature = "alloc_system_type", since = "1.28.0")]
 pub struct System;
 #[cfg(any(windows, unix, target_os = "redox"))]
 mod realloc_fallback {
@@ -96,7 +54,6 @@ mod platform {
     use MIN_ALIGN;
     use System;
     use core::alloc::{GlobalAlloc, Layout};
-    #[stable(feature = "alloc_system_type", since = "1.28.0")]
     unsafe impl GlobalAlloc for System {
         #[inline]
         unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
@@ -221,7 +178,6 @@ mod platform {
         };
         ptr as *mut u8
     }
-    #[stable(feature = "alloc_system_type", since = "1.28.0")]
     unsafe impl GlobalAlloc for System {
         #[inline]
         unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
@@ -254,89 +210,3 @@ mod platform {
         }
     }
 }
-// This is an implementation of a global allocator on the wasm32 platform when
-// emscripten is not in use. In that situation there's no actual runtime for us
-// to lean on for allocation, so instead we provide our own!
-//
-// The wasm32 instruction set has two instructions for getting the current
-// amount of memory and growing the amount of memory. These instructions are the
-// foundation on which we're able to build an allocator, so we do so! Note that
-// the instructions are also pretty "global" and this is the "global" allocator
-// after all!
-//
-// The current allocator here is the `dlmalloc` crate which we've got included
-// in the rust-lang/rust repository as a submodule. The crate is a port of
-// dlmalloc.c from C to Rust and is basically just so we can have "pure Rust"
-// for now which is currently technically required (can't link with C yet).
-//
-// The crate itself provides a global allocator which on wasm has no
-// synchronization as there are no threads!
-#[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
-mod platform {
-    extern crate dlmalloc;
-    use core::alloc::{GlobalAlloc, Layout};
-    use System;
-    static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::DLMALLOC_INIT;
-    #[stable(feature = "alloc_system_type", since = "1.28.0")]
-    unsafe impl GlobalAlloc for System {
-        #[inline]
-        unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
-            let _lock = lock::lock();
-            DLMALLOC.malloc(layout.size(), layout.align())
-        }
-        #[inline]
-        unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
-            let _lock = lock::lock();
-            DLMALLOC.calloc(layout.size(), layout.align())
-        }
-        #[inline]
-        unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
-            let _lock = lock::lock();
-            DLMALLOC.free(ptr, layout.size(), layout.align())
-        }
-        #[inline]
-        unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
-            let _lock = lock::lock();
-            DLMALLOC.realloc(ptr, layout.size(), layout.align(), new_size)
-        }
-    }
-    #[cfg(target_feature = "atomics")]
-    mod lock {
-        use core::arch::wasm32;
-        use core::sync::atomic::{AtomicI32, Ordering::SeqCst};
-        static LOCKED: AtomicI32 = AtomicI32::new(0);
-        pub struct DropLock;
-        pub fn lock() -> DropLock {
-            loop {
-                if LOCKED.swap(1, SeqCst) == 0 {
-                    return DropLock
-                }
-                unsafe {
-                    let r = wasm32::atomic::wait_i32(
-                        &LOCKED as *const AtomicI32 as *mut i32,
-                        1,  // expected value
-                        -1, // timeout
-                    );
-                    debug_assert!(r == 0 || r == 1);
-                }
-            }
-        }
-        impl Drop for DropLock {
-            fn drop(&mut self) {
-                let r = LOCKED.swap(0, SeqCst);
-                debug_assert_eq!(r, 1);
-                unsafe {
-                    wasm32::atomic::wake(
-                        &LOCKED as *const AtomicI32 as *mut i32,
-                        1, // only one thread
-                    );
-                }
-            }
-        }
-    }
-    #[cfg(not(target_feature = "atomics"))]
-    mod lock {
-        #[inline]
-        pub fn lock() {} // no atomics, no threads, that's easy!
-    }
-}
diff --git a/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs b/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs
index 0b0039a1370..ddeb752f93e 100644
--- a/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs
+++ b/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs
@@ -1,22 +1,12 @@
 // Adapted from rustc run-pass test suite
 
-#![feature(no_core, arbitrary_self_types, box_syntax)]
+#![feature(arbitrary_self_types, unsize, coerce_unsized, dispatch_from_dyn)]
 #![feature(rustc_attrs)]
 
-#![feature(start, lang_items)]
-#![no_core]
-
-extern crate mini_core;
-
-use mini_core::*;
-
-macro_rules! assert_eq {
-    ($l:expr, $r: expr) => {
-        if $l != $r {
-            panic(stringify!($l != $r));
-        }
-    }
-}
+use std::{
+    ops::{Deref, CoerceUnsized, DispatchFromDyn},
+    marker::Unsize,
+};
 
 struct Ptr<T: ?Sized>(Box<T>);
 
@@ -67,16 +57,13 @@ impl Trait for i32 {
     }
 }
 
-#[start]
-fn main(_: isize, _: *const *const u8) -> isize {
-    let pw = Ptr(box Wrapper(5)) as Ptr<Wrapper<dyn Trait>>;
+fn main() {
+    let pw = Ptr(Box::new(Wrapper(5))) as Ptr<Wrapper<dyn Trait>>;
     assert_eq!(pw.ptr_wrapper(), 5);
 
-    let wp = Wrapper(Ptr(box 6)) as Wrapper<Ptr<dyn Trait>>;
+    let wp = Wrapper(Ptr(Box::new(6))) as Wrapper<Ptr<dyn Trait>>;
     assert_eq!(wp.wrapper_ptr(), 6);
 
-    let wpw = Wrapper(Ptr(box Wrapper(7))) as Wrapper<Ptr<Wrapper<dyn Trait>>>;
+    let wpw = Wrapper(Ptr(Box::new(Wrapper(7)))) as Wrapper<Ptr<Wrapper<dyn Trait>>>;
     assert_eq!(wpw.wrapper_ptr_wrapper(), 7);
-
-    0
 }
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs
index 002ec7e2e3d..c4834c80408 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs
@@ -365,6 +365,22 @@ impl <T: PartialEq> PartialEq for Option<T> {
     }
 }
 
+#[lang = "shl"]
+pub trait Shl<RHS = Self> {
+    type Output;
+
+    #[must_use]
+    fn shl(self, rhs: RHS) -> Self::Output;
+}
+
+impl Shl for u128 {
+    type Output = u128;
+
+    fn shl(self, rhs: u128) -> u128 {
+        self << rhs
+    }
+}
+
 #[lang = "neg"]
 pub trait Neg {
     type Output;
@@ -605,6 +621,7 @@ struct PanicLocation {
 }
 
 #[no_mangle]
+#[cfg(not(windows))]
 pub fn get_tls() -> u8 {
     #[thread_local]
     static A: u8 = 42;
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
index 4a8375afac3..ea37ca98b59 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
@@ -1,7 +1,4 @@
-#![feature(
-    no_core, start, lang_items, box_syntax, never_type, linkage,
-    extern_types, thread_local
-)]
+#![feature(no_core, lang_items, box_syntax, never_type, linkage, extern_types, thread_local)]
 #![no_core]
 #![allow(dead_code, non_camel_case_types)]
 
@@ -239,7 +236,7 @@ fn main() {
 
     assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42);
 
-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
     {
         extern {
             #[linkage = "extern_weak"]
@@ -264,6 +261,9 @@ fn main() {
     assert_eq!(f2 as i8, -128);
     assert_eq!(f2 as u8, 0);
 
+    let amount = 0;
+    assert_eq!(1u128 << amount, 1);
+
     static ANOTHER_STATIC: &u8 = &A_STATIC;
     assert_eq!(*ANOTHER_STATIC, 42);
 
@@ -289,7 +289,7 @@ fn main() {
 
     from_decimal_string();
 
-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
     test_tls();
 
     #[cfg(all(not(jit), target_os = "linux"))]
diff --git a/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch b/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch
index 3eb10069ada..8cfffe580a1 100644
--- a/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch
@@ -119,21 +119,5 @@ index 6609bc3..241b497 100644
  
  #[test]
  #[should_panic(expected = "index 0 greater than length of slice")]
-diff --git a/library/core/tests/num/ops.rs b/library/core/tests/num/ops.rs
-index 9979cc8..d5d1d83 100644
---- a/library/core/tests/num/ops.rs
-+++ b/library/core/tests/num/ops.rs
-@@ -238,7 +238,7 @@ macro_rules! test_shift_assign {
-         }
-     };
- }
--test_shift!(test_shl_defined, Shl::shl);
--test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
--test_shift!(test_shr_defined, Shr::shr);
--test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
-+//test_shift!(test_shl_defined, Shl::shl);
-+//test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
-+//test_shift!(test_shr_defined, Shr::shr);
-+//test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
 --
 2.21.0 (Apple Git-122)
diff --git a/compiler/rustc_codegen_cranelift/patches/0027-Disable-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-Disable-128bit-atomic-operations.patch
new file mode 100644
index 00000000000..32e59309690
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/patches/0027-Disable-128bit-atomic-operations.patch
@@ -0,0 +1,103 @@
+From 894e07dfec2624ba539129b1c1d63e1d7d812bda Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Thu, 18 Feb 2021 18:45:28 +0100
+Subject: [PATCH] Disable 128bit atomic operations
+
+Cranelift doesn't support them yet
+---
+ library/core/src/sync/atomic.rs | 38 ---------------------------------
+ library/core/tests/atomic.rs    |  4 ----
+ library/std/src/panic.rs        |  6 ------
+ 3 files changed, 48 deletions(-)
+
+diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs
+index 81c9e1d..65c9503 100644
+--- a/library/core/src/sync/atomic.rs
++++ b/library/core/src/sync/atomic.rs
+@@ -2228,44 +2228,6 @@ atomic_int! {
+     "AtomicU64::new(0)",
+     u64 AtomicU64 ATOMIC_U64_INIT
+ }
+-#[cfg(target_has_atomic_load_store = "128")]
+-atomic_int! {
+-    cfg(target_has_atomic = "128"),
+-    cfg(target_has_atomic_equal_alignment = "128"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    "i128",
+-    "#![feature(integer_atomics)]\n\n",
+-    atomic_min, atomic_max,
+-    16,
+-    "AtomicI128::new(0)",
+-    i128 AtomicI128 ATOMIC_I128_INIT
+-}
+-#[cfg(target_has_atomic_load_store = "128")]
+-atomic_int! {
+-    cfg(target_has_atomic = "128"),
+-    cfg(target_has_atomic_equal_alignment = "128"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    "u128",
+-    "#![feature(integer_atomics)]\n\n",
+-    atomic_umin, atomic_umax,
+-    16,
+-    "AtomicU128::new(0)",
+-    u128 AtomicU128 ATOMIC_U128_INIT
+-}
+ 
+ macro_rules! atomic_int_ptr_sized {
+     ( $($target_pointer_width:literal $align:literal)* ) => { $(
+diff --git a/library/core/tests/atomic.rs b/library/core/tests/atomic.rs
+index 2d1e449..cb6da5d 100644
+--- a/library/core/tests/atomic.rs
++++ b/library/core/tests/atomic.rs
+@@ -145,10 +145,6 @@ fn atomic_alignment() {
+     assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>());
+     #[cfg(target_has_atomic = "64")]
+     assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>());
+     #[cfg(target_has_atomic = "ptr")]
+     assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
+     #[cfg(target_has_atomic = "ptr")]
+diff --git a/library/std/src/panic.rs b/library/std/src/panic.rs
+index 89a822a..779fd88 100644
+--- a/library/std/src/panic.rs
++++ b/library/std/src/panic.rs
+@@ -279,9 +279,6 @@ impl RefUnwindSafe for atomic::AtomicI32 {}
+ #[cfg(target_has_atomic_load_store = "64")]
+ #[stable(feature = "integer_atomics_stable", since = "1.34.0")]
+ impl RefUnwindSafe for atomic::AtomicI64 {}
+-#[cfg(target_has_atomic_load_store = "128")]
+-#[unstable(feature = "integer_atomics", issue = "32976")]
+-impl RefUnwindSafe for atomic::AtomicI128 {}
+ 
+ #[cfg(target_has_atomic_load_store = "ptr")]
+ #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")]
+@@ -298,9 +295,6 @@ impl RefUnwindSafe for atomic::AtomicU32 {}
+ #[cfg(target_has_atomic_load_store = "64")]
+ #[stable(feature = "integer_atomics_stable", since = "1.34.0")]
+ impl RefUnwindSafe for atomic::AtomicU64 {}
+-#[cfg(target_has_atomic_load_store = "128")]
+-#[unstable(feature = "integer_atomics", issue = "32976")]
+-impl RefUnwindSafe for atomic::AtomicU128 {}
+ 
+ #[cfg(target_has_atomic_load_store = "8")]
+ #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")]
+-- 
+2.26.2.7.g19db9cfb68
+
diff --git a/compiler/rustc_codegen_cranelift/prepare.sh b/compiler/rustc_codegen_cranelift/prepare.sh
index 08e7cb18029..64c097261c9 100755
--- a/compiler/rustc_codegen_cranelift/prepare.sh
+++ b/compiler/rustc_codegen_cranelift/prepare.sh
@@ -1,7 +1,6 @@
-#!/bin/bash --verbose
+#!/usr/bin/env bash
 set -e
 
-rustup component add rust-src rustc-dev llvm-tools-preview
 ./build_sysroot/prepare_sysroot_src.sh
 cargo install hyperfine || echo "Skipping hyperfine install"
 
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index a08f00d19c2..2917fc7ee39 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1 +1,3 @@
-nightly-2021-01-30
+[toolchain]
+channel = "nightly-2021-03-29"
+components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/compiler/rustc_codegen_cranelift/rustfmt.toml b/compiler/rustc_codegen_cranelift/rustfmt.toml
new file mode 100644
index 00000000000..2bd8f7d1bc1
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/rustfmt.toml
@@ -0,0 +1,4 @@
+# Matches rustfmt.toml of rustc
+version = "Two"
+use_small_heuristics = "Max"
+merge_derives = false
diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo.sh b/compiler/rustc_codegen_cranelift/scripts/cargo.sh
index a3d6d303057..1daa5a78f7b 100755
--- a/compiler/rustc_codegen_cranelift/scripts/cargo.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/cargo.sh
@@ -1,10 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 dir=$(dirname "$0")
 source "$dir/config.sh"
 
 # read nightly compiler from rust-toolchain file
-TOOLCHAIN=$(cat "$dir/rust-toolchain")
+TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/")
 
 cmd=$1
 shift || true
diff --git a/compiler/rustc_codegen_cranelift/scripts/config.sh b/compiler/rustc_codegen_cranelift/scripts/config.sh
index 834708aa9a6..99b302ee1d9 100644
--- a/compiler/rustc_codegen_cranelift/scripts/config.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/config.sh
@@ -2,15 +2,7 @@
 
 set -e
 
-unamestr=$(uname)
-if [[ "$unamestr" == 'Linux' ]]; then
-   dylib_ext='so'
-elif [[ "$unamestr" == 'Darwin' ]]; then
-   dylib_ext='dylib'
-else
-   echo "Unsupported os"
-   exit 1
-fi
+dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -)
 
 if echo "$RUSTC_WRAPPER" | grep sccache; then
 echo
@@ -24,10 +16,10 @@ dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 export RUSTC=$dir"/bin/cg_clif"
 
 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
-'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
+'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir
 
-# FIXME remove once the atomic shim is gone
-if [[ "$unamestr" == 'Darwin' ]]; then
+# FIXME fix `#[linkage = "extern_weak"]` without this
+if [[ "$(uname)" == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
index 430f5c469b4..fa7557653d8 100755
--- a/compiler/rustc_codegen_cranelift/scripts/rustup.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
@@ -8,7 +8,7 @@ case $1 in
 
         echo "=> Installing new nightly"
         rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
-        echo "nightly-${TOOLCHAIN}" > rust-toolchain
+        sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
         rustup component add rustfmt || true
 
         echo "=> Uninstalling all old nighlies"
diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
new file mode 100644
index 00000000000..e8bedf625f7
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+set -e
+
+./build.sh
+source build/config.sh
+
+echo "[SETUP] Rust fork"
+git clone https://github.com/rust-lang/rust.git || true
+pushd rust
+git fetch
+git checkout -- .
+git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
+
+git apply - <<EOF
+diff --git a/Cargo.toml b/Cargo.toml
+index 5bd1147cad5..10d68a2ff14 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+ rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
+ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+
++compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
++
+ [patch."https://github.com/rust-lang/rust-clippy"]
+ clippy_lints = { path = "src/tools/clippy/clippy_lints" }
+diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
+index 23e689fcae7..5f077b765b6 100644
+--- a/compiler/rustc_data_structures/Cargo.toml
++++ b/compiler/rustc_data_structures/Cargo.toml
+@@ -32,7 +32,6 @@ tempfile = "3.0.5"
+
+ [dependencies.parking_lot]
+ version = "0.11"
+-features = ["nightly"]
+
+ [target.'cfg(windows)'.dependencies]
+ winapi = { version = "0.3", features = ["fileapi", "psapi"] }
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
++++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
+
+ [dev-dependencies]
+ rand = "0.7"
+EOF
+
+cat > config.toml <<EOF
+[llvm]
+ninja = false
+
+[build]
+rustc = "$(pwd)/../build/bin/cg_clif"
+cargo = "$(rustup which cargo)"
+full-bootstrap = true
+local-rebuild = true
+
+[rust]
+codegen-backends = ["cranelift"]
+deny-warnings = false
+EOF
+popd
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
index db69541b226..791d457993d 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
@@ -1,62 +1,12 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 cd "$(dirname "$0")/../"
 
-./build.sh
-source build/config.sh
+source ./scripts/setup_rust_fork.sh
 
 echo "[TEST] Bootstrap of rustc"
-git clone https://github.com/rust-lang/rust.git || true
 pushd rust
-git fetch
-git checkout -- .
-git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
-
-git apply - <<EOF
-diff --git a/.gitmodules b/.gitmodules
-index 984113151de..c1e9d960d56 100644
---- a/.gitmodules
-+++ b/.gitmodules
-@@ -34,10 +34,6 @@
- [submodule "src/doc/edition-guide"]
- 	path = src/doc/edition-guide
- 	url = https://github.com/rust-lang/edition-guide.git
--[submodule "src/llvm-project"]
--	path = src/llvm-project
--	url = https://github.com/rust-lang/llvm-project.git
--	branch = rustc/11.0-2020-10-12
- [submodule "src/doc/embedded-book"]
- 	path = src/doc/embedded-book
- 	url = https://github.com/rust-embedded/book.git
-diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
-index 23e689fcae7..5f077b765b6 100644
---- a/compiler/rustc_data_structures/Cargo.toml
-+++ b/compiler/rustc_data_structures/Cargo.toml
-@@ -32,7 +32,6 @@ tempfile = "3.0.5"
-
- [dependencies.parking_lot]
- version = "0.11"
--features = ["nightly"]
-
- [target.'cfg(windows)'.dependencies]
- winapi = { version = "0.3", features = ["fileapi", "psapi"] }
-EOF
-
-cat > config.toml <<EOF
-[llvm]
-ninja = false
-
-[build]
-rustc = "$(pwd)/../build/bin/cg_clif"
-cargo = "$(rustup which cargo)"
-full-bootstrap = true
-local-rebuild = true
-
-[rust]
-codegen-backends = ["cranelift"]
-EOF
-
 rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src}
 cp ../Cargo.* compiler/rustc_codegen_cranelift/
 cp -r ../src compiler/rustc_codegen_cranelift/src
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
new file mode 100755
index 00000000000..fbc3feceec7
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+cd $(dirname "$0")/../
+
+source ./scripts/setup_rust_fork.sh
+
+echo "[TEST] Test suite of rustc"
+pushd rust
+
+cargo install ripgrep
+
+rm -r src/test/ui/{extern/,panics/,unsized-locals/,thinlto/,simd*,*lto*.rs,linkage*,unwind-*.rs} || true
+for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto" src/test/ui); do
+  rm $test
+done
+
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" src/test/ui); do
+  rm $test
+done
+
+git checkout -- src/test/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
+
+# these all depend on unwinding support
+rm src/test/ui/backtrace.rs
+rm src/test/ui/array-slice-vec/box-of-array-of-drop-*.rs
+rm src/test/ui/array-slice-vec/slice-panic-*.rs
+rm src/test/ui/array-slice-vec/nested-vec-3.rs
+rm src/test/ui/cleanup-rvalue-temp-during-incomplete-alloc.rs
+rm src/test/ui/issues/issue-26655.rs
+rm src/test/ui/issues/issue-29485.rs
+rm src/test/ui/issues/issue-30018-panic.rs
+rm src/test/ui/multi-panic.rs
+rm src/test/ui/sepcomp/sepcomp-unwind.rs
+rm src/test/ui/structs-enums/unit-like-struct-drop-run.rs
+rm src/test/ui/terminate-in-initializer.rs
+rm src/test/ui/threads-sendsync/task-stderr.rs
+rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
+rm src/test/ui/drop/drop-trait-enum.rs
+rm src/test/ui/numbers-arithmetic/issue-8460.rs
+
+rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
+rm src/test/ui/init-large-type.rs # same
+rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
+rm src/test/ui/issues/issue-33992.rs # unsupported linkages
+rm src/test/ui/issues/issue-51947.rs # same
+rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
+rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
+rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
+rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
+rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
+rm src/test/ui/generator/size-moved-locals.rs # same
+rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
+rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-type.rs # "Cannot run dynamic test fn out-of-process"
+rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
+
+rm src/test/incremental/hashes/inline_asm.rs # inline asm
+rm src/test/incremental/issue-72386.rs # same
+rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind
+rm src/test/incremental/issue-49482.rs # same
+rm src/test/incremental/issue-54059.rs # same
+rm src/test/incremental/lto.rs # requires lto
+
+rm src/test/pretty/asm.rs # inline asm
+rm src/test/pretty/raw-str-nonexpr.rs # same
+
+rm -r src/test/run-pass-valgrind/unsized-locals
+
+rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
+rm src/test/ui/json-bom-plus-crlf.rs # same
+rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
+
+rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition
+rm src/test/ui/cfg/cfg-panic.rs
+rm src/test/ui/default-alloc-error-hook.rs
+rm -r src/test/ui/hygiene/
+
+rm -r src/test/ui/polymorphization/ # polymorphization not yet supported
+rm src/test/codegen-units/polymorphization/unused_type_parameters.rs # same
+
+rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
+rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
+rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
+
+echo "[TEST] rustc test suite"
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui}
+popd
diff --git a/compiler/rustc_codegen_cranelift/scripts/tests.sh b/compiler/rustc_codegen_cranelift/scripts/tests.sh
index d37b57babe6..3afcea8f06b 100755
--- a/compiler/rustc_codegen_cranelift/scripts/tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/tests.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
@@ -27,13 +27,16 @@ function no_sysroot_tests() {
     $MY_RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd
     # (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd
+}
 
+function base_sysroot_tests() {
     echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
     $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
-}
 
-function base_sysroot_tests() {
+    echo "[AOT] alloc_system"
+    $MY_RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
+
     echo "[AOT] alloc_example"
     $MY_RUSTC example/alloc_example.rs --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/alloc_example
@@ -68,14 +71,20 @@ function base_sysroot_tests() {
     echo "[AOT] mod_bench"
     $MY_RUSTC example/mod_bench.rs --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/mod_bench
+}
 
+function extended_sysroot_tests() {
     pushd rand
-    rm -r ./target || true
-    ../build/cargo.sh test --workspace
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-random/rand"
+        ../build/cargo.sh test --workspace
+    else
+        echo "[AOT] rust-random/rand"
+        ../build/cargo.sh build --workspace --target $TARGET_TRIPLE --tests
+    fi
     popd
-}
 
-function extended_sysroot_tests() {
     pushd simple-raytracer
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[BENCH COMPILE] ebobby/simple-raytracer"
@@ -89,27 +98,40 @@ function extended_sysroot_tests() {
     else
         echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
         echo "[COMPILE] ebobby/simple-raytracer"
-        ../cargo.sh build
+        ../build/cargo.sh build --target $TARGET_TRIPLE
         echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
     fi
     popd
 
     pushd build_sysroot/sysroot_src/library/core/tests
     echo "[TEST] libcore"
-    rm -r ./target || true
-    ../../../../../build/cargo.sh test
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        ../../../../../build/cargo.sh test
+    else
+        ../../../../../build/cargo.sh build --target $TARGET_TRIPLE --tests
+    fi
     popd
 
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../build/cargo.sh clean
+    cargo clean
     # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../build/cargo.sh build --example shootout-regex-dna
-    cat examples/regexdna-input.txt | ../build/cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt
-    diff -u res.txt examples/regexdna-output.txt
+    ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        cat examples/regexdna-input.txt \
+            | ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \
+            | grep -v "Spawned thread" > res.txt
+        diff -u res.txt examples/regexdna-output.txt
+    fi
 
-    echo "[TEST] rust-lang/regex tests"
-    ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-lang/regex tests"
+        ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    else
+        echo "[AOT] rust-lang/regex tests"
+        ../build/cargo.sh build --tests --target $TARGET_TRIPLE
+    fi
     popd
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
index 9aab45b62e2..5fbaed7283a 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
@@ -10,14 +10,16 @@ use cranelift_codegen::entity::EntityRef;
 
 use crate::prelude::*;
 
-pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    fx.add_global_comment(
-        "kind  loc.idx   param    pass mode                            ty".to_string(),
-    );
+pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(
+            "kind  loc.idx   param    pass mode                            ty".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_arg_comment<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     kind: &str,
     local: Option<mir::Local>,
     local_field: Option<usize>,
@@ -25,6 +27,10 @@ pub(super) fn add_arg_comment<'tcx>(
     arg_abi_mode: PassMode,
     arg_layout: TyAndLayout<'tcx>,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
+
     let local = if let Some(local) = local {
         Cow::Owned(format!("{:?}", local))
     } else {
@@ -42,11 +48,7 @@ pub(super) fn add_arg_comment<'tcx>(
         [param_a, param_b] => Cow::Owned(format!("= {:?},{:?}", param_a, param_b)),
         params => Cow::Owned(format!(
             "= {}",
-            params
-                .iter()
-                .map(ToString::to_string)
-                .collect::<Vec<_>>()
-                .join(",")
+            params.iter().map(ToString::to_string).collect::<Vec<_>>().join(",")
         )),
     };
 
@@ -62,27 +64,26 @@ pub(super) fn add_arg_comment<'tcx>(
     ));
 }
 
-pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    fx.add_global_comment(String::new());
-    fx.add_global_comment(
-        "kind  local ty                              size align (abi,pref)".to_string(),
-    );
+pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(String::new());
+        fx.add_global_comment(
+            "kind  local ty                              size align (abi,pref)".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_local_place_comments<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     place: CPlace<'tcx>,
     local: Local,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
     let TyAndLayout { ty, layout } = place.layout();
-    let rustc_target::abi::Layout {
-        size,
-        align,
-        abi: _,
-        variants: _,
-        fields: _,
-        largest_niche: _,
-    } = layout;
+    let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
+        layout;
 
     let (kind, extra) = match *place.inner() {
         CPlaceInner::Var(place_local, var) => {
@@ -91,10 +92,7 @@ pub(super) fn add_local_place_comments<'tcx>(
         }
         CPlaceInner::VarPair(place_local, var1, var2) => {
             assert_eq!(local, place_local);
-            (
-                "ssa",
-                Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())),
-            )
+            ("ssa", Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())))
         }
         CPlaceInner::VarLane(_local, _var, _lane) => unreachable!(),
         CPlaceInner::Addr(ptr, meta) => {
@@ -103,19 +101,16 @@ pub(super) fn add_local_place_comments<'tcx>(
             } else {
                 Cow::Borrowed("")
             };
-            match ptr.base_and_offset() {
-                (crate::pointer::PointerBase::Addr(addr), offset) => (
-                    "reuse",
-                    format!("storage={}{}{}", addr, offset, meta).into(),
-                ),
-                (crate::pointer::PointerBase::Stack(stack_slot), offset) => (
-                    "stack",
-                    format!("storage={}{}{}", stack_slot, offset, meta).into(),
-                ),
-                (crate::pointer::PointerBase::Dangling(align), offset) => (
-                    "zst",
-                    format!("align={},offset={}", align.bytes(), offset).into(),
-                ),
+            match ptr.debug_base_and_offset() {
+                (crate::pointer::PointerBase::Addr(addr), offset) => {
+                    ("reuse", format!("storage={}{}{}", addr, offset, meta).into())
+                }
+                (crate::pointer::PointerBase::Stack(stack_slot), offset) => {
+                    ("stack", format!("storage={}{}{}", stack_slot, offset, meta).into())
+                }
+                (crate::pointer::PointerBase::Dangling(align), offset) => {
+                    ("zst", format!("align={},offset={}", align.bytes(), offset).into())
+                }
             }
         }
     };
@@ -128,11 +123,7 @@ pub(super) fn add_local_place_comments<'tcx>(
         size.bytes(),
         align.abi.bytes(),
         align.pref.bytes(),
-        if extra.is_empty() {
-            ""
-        } else {
-            "              "
-        },
+        if extra.is_empty() { "" } else { "              " },
         extra,
     ));
 }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index b2647e6c8d3..0e7829eaa26 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -1,6 +1,5 @@
 //! Handling of everything related to the calling convention. Also fills `fx.local_map`.
 
-#[cfg(debug_assertions)]
 mod comments;
 mod pass_mode;
 mod returning;
@@ -38,25 +37,15 @@ fn clif_sig_from_fn_abi<'tcx>(
         | Conv::X86VectorCall
         | Conv::AmdGpuKernel
         | Conv::AvrInterrupt
-        | Conv::AvrNonBlockingInterrupt => {
-            todo!("{:?}", fn_abi.conv)
-        }
+        | Conv::AvrNonBlockingInterrupt => todo!("{:?}", fn_abi.conv),
     };
-    let inputs = fn_abi
-        .args
-        .iter()
-        .map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter())
-        .flatten();
+    let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
 
     let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
     // Sometimes the first param is an pointer to the place where the return value needs to be stored.
     let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect();
 
-    Signature {
-        params,
-        returns,
-        call_conv,
-    }
+    Signature { params, returns, call_conv }
 }
 
 pub(crate) fn get_function_sig<'tcx>(
@@ -65,37 +54,29 @@ pub(crate) fn get_function_sig<'tcx>(
     inst: Instance<'tcx>,
 ) -> Signature {
     assert!(!inst.substs.needs_infer());
-    clif_sig_from_fn_abi(
-        tcx,
-        triple,
-        &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]),
-    )
+    clif_sig_from_fn_abi(tcx, triple, &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]))
 }
 
 /// Instance must be monomorphized
 pub(crate) fn import_function<'tcx>(
     tcx: TyCtxt<'tcx>,
-    module: &mut impl Module,
+    module: &mut dyn Module,
     inst: Instance<'tcx>,
 ) -> FuncId {
     let name = tcx.symbol_name(inst).name.to_string();
     let sig = get_function_sig(tcx, module.isa().triple(), inst);
-    module
-        .declare_function(&name, Linkage::Import, &sig)
-        .unwrap()
+    module.declare_function(&name, Linkage::Import, &sig).unwrap()
 }
 
-impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
+impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     /// Instance must be monomorphized
     pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
-        let func_id = import_function(self.tcx, &mut self.cx.module, inst);
-        let func_ref = self
-            .cx
-            .module
-            .declare_func_in_func(func_id, &mut self.bcx.func);
+        let func_id = import_function(self.tcx, self.cx.module, inst);
+        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
 
-        #[cfg(debug_assertions)]
-        self.add_comment(func_ref, format!("{:?}", inst));
+        if self.clif_comments.enabled() {
+            self.add_comment(func_ref, format!("{:?}", inst));
+        }
 
         func_ref
     }
@@ -107,23 +88,11 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
         returns: Vec<AbiParam>,
         args: &[Value],
     ) -> &[Value] {
-        let sig = Signature {
-            params,
-            returns,
-            call_conv: CallConv::triple_default(self.triple()),
-        };
-        let func_id = self
-            .cx
-            .module
-            .declare_function(&name, Linkage::Import, &sig)
-            .unwrap();
-        let func_ref = self
-            .cx
-            .module
-            .declare_func_in_func(func_id, &mut self.bcx.func);
+        let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) };
+        let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
+        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
         let call_inst = self.bcx.ins().call(func_ref, args);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
             self.add_comment(call_inst, format!("easy_call {}", name));
         }
         let results = self.bcx.inst_results(call_inst);
@@ -140,17 +109,12 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
         let (input_tys, args): (Vec<_>, Vec<_>) = args
             .iter()
             .map(|arg| {
-                (
-                    AbiParam::new(self.clif_type(arg.layout().ty).unwrap()),
-                    arg.load_scalar(self),
-                )
+                (AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self))
             })
             .unzip();
         let return_layout = self.layout_of(return_ty);
         let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
-            tup.types()
-                .map(|ty| AbiParam::new(self.clif_type(ty).unwrap()))
-                .collect()
+            tup.types().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
         } else {
             vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
         };
@@ -169,7 +133,7 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
 
 /// Make a [`CPlace`] capable of holding value of the specified type.
 fn make_local_place<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     local: Local,
     layout: TyAndLayout<'tcx>,
     is_ssa: bool,
@@ -184,16 +148,12 @@ fn make_local_place<'tcx>(
         CPlace::new_stack_slot(fx, layout)
     };
 
-    #[cfg(debug_assertions)]
     self::comments::add_local_place_comments(fx, place, local);
 
     place
 }
 
-pub(crate) fn codegen_fn_prelude<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    start_block: Block,
-) {
+pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_block: Block) {
     fx.bcx.append_block_params_for_function_params(start_block);
 
     fx.bcx.switch_to_block(start_block);
@@ -201,16 +161,9 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
 
     let ssa_analyzed = crate::analyze::analyze(fx);
 
-    #[cfg(debug_assertions)]
     self::comments::add_args_header_comment(fx);
 
-    let mut block_params_iter = fx
-        .bcx
-        .func
-        .dfg
-        .block_params(start_block)
-        .to_vec()
-        .into_iter();
+    let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
     let ret_place =
         self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter);
     assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE);
@@ -272,7 +225,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
     fx.fn_abi = Some(fn_abi);
     assert!(block_params_iter.next().is_none(), "arg_value left behind");
 
-    #[cfg(debug_assertions)]
     self::comments::add_locals_header_comment(fx);
 
     for (local, arg_kind, ty) in func_params {
@@ -286,10 +238,10 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
             if let Some((addr, meta)) = val.try_to_ptr() {
                 let local_decl = &fx.mir.local_decls[local];
                 //                       v this ! is important
-                let internally_mutable = !val.layout().ty.is_freeze(
-                    fx.tcx.at(local_decl.source_info.span),
-                    ParamEnv::reveal_all(),
-                );
+                let internally_mutable = !val
+                    .layout()
+                    .ty
+                    .is_freeze(fx.tcx.at(local_decl.source_info.span), ParamEnv::reveal_all());
                 if local_decl.mutability == mir::Mutability::Not && !internally_mutable {
                     // We wont mutate this argument, so it is fine to borrow the backing storage
                     // of this argument, to prevent a copy.
@@ -300,7 +252,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
                         CPlace::for_ptr(addr, val.layout())
                     };
 
-                    #[cfg(debug_assertions)]
                     self::comments::add_local_place_comments(fx, place, local);
 
                     assert_eq!(fx.local_map.push(place), local);
@@ -321,9 +272,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
             ArgKind::Spread(params) => {
                 for (i, param) in params.into_iter().enumerate() {
                     if let Some(param) = param {
-                        place
-                            .place_field(fx, mir::Field::new(i))
-                            .write_cvalue(fx, param);
+                        place.place_field(fx, mir::Field::new(i)).write_cvalue(fx, param);
                     }
                 }
             }
@@ -340,13 +289,11 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
         assert_eq!(fx.local_map.push(place), local);
     }
 
-    fx.bcx
-        .ins()
-        .jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
+    fx.bcx.ins().jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
 }
 
 pub(crate) fn codegen_terminator_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     span: Span,
     current_block: Block,
     func: &Operand<'tcx>,
@@ -354,9 +301,8 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     destination: Option<(Place<'tcx>, BasicBlock)>,
 ) {
     let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
-    let fn_sig = fx
-        .tcx
-        .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
+    let fn_sig =
+        fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
 
     let destination = destination.map(|(place, bb)| (codegen_place(fx, place), bb));
 
@@ -404,20 +350,11 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     let fn_abi = if let Some(instance) = instance {
         FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
     } else {
-        FnAbi::of_fn_ptr(
-            &RevealAllLayoutCx(fx.tcx),
-            fn_ty.fn_sig(fx.tcx),
-            &extra_args,
-        )
+        FnAbi::of_fn_ptr(&RevealAllLayoutCx(fx.tcx), fn_ty.fn_sig(fx.tcx), &extra_args)
     };
 
     let is_cold = instance
-        .map(|inst| {
-            fx.tcx
-                .codegen_fn_attrs(inst.def_id())
-                .flags
-                .contains(CodegenFnAttrFlags::COLD)
-        })
+        .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD))
         .unwrap_or(false);
     if is_cold {
         fx.cold_blocks.insert(current_block);
@@ -441,9 +378,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         }
         args
     } else {
-        args.iter()
-            .map(|arg| codegen_operand(fx, arg))
-            .collect::<Vec<_>>()
+        args.iter().map(|arg| codegen_operand(fx, arg)).collect::<Vec<_>>()
     };
 
     //   | indirect call target
@@ -451,12 +386,8 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     //   v         v
     let (func_ref, first_arg) = match instance {
         // Trait object call
-        Some(Instance {
-            def: InstanceDef::Virtual(_, idx),
-            ..
-        }) => {
-            #[cfg(debug_assertions)]
-            {
+        Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
+            if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(
                     nop_inst,
@@ -477,8 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
         // Indirect call
         None => {
-            #[cfg(debug_assertions)]
-            {
+            if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(nop_inst, "indirect call");
             }
@@ -511,10 +441,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 )
                 .collect::<Vec<_>>();
 
-            if instance
-                .map(|inst| inst.def.requires_caller_location(fx.tcx))
-                .unwrap_or(false)
-            {
+            if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) {
                 // Pass the caller location for `#[track_caller]`.
                 let caller_location = fx.get_caller_location(span);
                 call_args.extend(
@@ -542,11 +469,8 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
     // FIXME find a cleaner way to support varargs
     if fn_sig.c_variadic {
-        if fn_sig.abi != Abi::C {
-            fx.tcx.sess.span_fatal(
-                span,
-                &format!("Variadic call for non-C abi {:?}", fn_sig.abi),
-            );
+        if !matches!(fn_sig.abi, Abi::C { .. }) {
+            fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
         }
         let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
         let abi_params = call_args
@@ -555,9 +479,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 let ty = fx.bcx.func.dfg.value_type(arg);
                 if !ty.is_int() {
                     // FIXME set %al to upperbound on float args once floats are supported
-                    fx.tcx
-                        .sess
-                        .span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
+                    fx.tcx.sess.span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
                 }
                 AbiParam::new(ty)
             })
@@ -574,7 +496,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 }
 
 pub(crate) fn codegen_drop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     span: Span,
     drop_place: CPlace<'tcx>,
 ) {
@@ -611,10 +533,7 @@ pub(crate) fn codegen_drop<'tcx>(
                     fx,
                     fx.layout_of(fx.tcx.mk_ref(
                         &ty::RegionKind::ReErased,
-                        TypeAndMut {
-                            ty,
-                            mutbl: crate::rustc_hir::Mutability::Mut,
-                        },
+                        TypeAndMut { ty, mutbl: crate::rustc_hir::Mutability::Mut },
                     )),
                 );
                 let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0]);
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 1202c23dbe7..7c275965199 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -71,12 +71,7 @@ fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> {
         .prefix
         .iter()
         .flatten()
-        .map(|&kind| {
-            reg_to_abi_param(Reg {
-                kind,
-                size: cast.prefix_chunk_size,
-            })
-        })
+        .map(|&kind| reg_to_abi_param(Reg { kind, size: cast.prefix_chunk_size }))
         .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
         .collect::<SmallVec<_>>();
 
@@ -98,12 +93,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         match self.mode {
             PassMode::Ignore => smallvec![],
             PassMode::Direct(attrs) => match &self.layout.abi {
-                Abi::Scalar(scalar) => {
-                    smallvec![apply_arg_attrs_to_abi_param(
-                        AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())),
-                        attrs
-                    )]
-                }
+                Abi::Scalar(scalar) => smallvec![apply_arg_attrs_to_abi_param(
+                    AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())),
+                    attrs
+                )],
                 Abi::Vector { .. } => {
                     let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
                     smallvec![AbiParam::new(vector_ty)]
@@ -122,11 +115,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 _ => unreachable!("{:?}", self.layout.abi),
             },
             PassMode::Cast(cast) => cast_target_to_abi_params(cast),
-            PassMode::Indirect {
-                attrs,
-                extra_attrs: None,
-                on_stack,
-            } => {
+            PassMode::Indirect { attrs, extra_attrs: None, on_stack } => {
                 if on_stack {
                     let size = u32::try_from(self.layout.size.bytes()).unwrap();
                     smallvec![apply_arg_attrs_to_abi_param(
@@ -134,17 +123,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                         attrs
                     )]
                 } else {
-                    smallvec![apply_arg_attrs_to_abi_param(
-                        AbiParam::new(pointer_ty(tcx)),
-                        attrs
-                    )]
+                    smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)]
                 }
             }
-            PassMode::Indirect {
-                attrs,
-                extra_attrs: Some(extra_attrs),
-                on_stack,
-            } => {
+            PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => {
                 assert!(!on_stack);
                 smallvec![
                     apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs),
@@ -158,10 +140,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         match self.mode {
             PassMode::Ignore => (None, vec![]),
             PassMode::Direct(_) => match &self.layout.abi {
-                Abi::Scalar(scalar) => (
-                    None,
-                    vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))],
-                ),
+                Abi::Scalar(scalar) => {
+                    (None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))])
+                }
                 Abi::Vector { .. } => {
                     let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
                     (None, vec![AbiParam::new(vector_ty)])
@@ -177,31 +158,19 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 _ => unreachable!("{:?}", self.layout.abi),
             },
             PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()),
-            PassMode::Indirect {
-                attrs: _,
-                extra_attrs: None,
-                on_stack,
-            } => {
+            PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => {
                 assert!(!on_stack);
-                (
-                    Some(AbiParam::special(
-                        pointer_ty(tcx),
-                        ArgumentPurpose::StructReturn,
-                    )),
-                    vec![],
-                )
+                (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
+            }
+            PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
+                unreachable!("unsized return value")
             }
-            PassMode::Indirect {
-                attrs: _,
-                extra_attrs: Some(_),
-                on_stack: _,
-            } => unreachable!("unsized return value"),
         }
     }
 }
 
 pub(super) fn to_casted_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     arg: CValue<'tcx>,
     cast: CastTarget,
 ) -> SmallVec<[Value; 2]> {
@@ -211,9 +180,7 @@ pub(super) fn to_casted_value<'tcx>(
     cast_target_to_abi_params(cast)
         .into_iter()
         .map(|param| {
-            let val = ptr
-                .offset_i64(fx, offset)
-                .load(fx, param.value_type, MemFlags::new());
+            let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new());
             offset += i64::from(param.value_type.bytes());
             val
         })
@@ -221,16 +188,13 @@ pub(super) fn to_casted_value<'tcx>(
 }
 
 pub(super) fn from_casted_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     block_params: &[Value],
     layout: TyAndLayout<'tcx>,
     cast: CastTarget,
 ) -> CValue<'tcx> {
     let abi_params = cast_target_to_abi_params(cast);
-    let abi_param_size: u32 = abi_params
-        .iter()
-        .map(|param| param.value_type.bytes())
-        .sum();
+    let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
     let layout_size = u32::try_from(layout.size.bytes()).unwrap();
     let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
         kind: StackSlotKind::ExplicitSlot,
@@ -244,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>(
     });
     let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
     let mut offset = 0;
-    let mut block_params_iter = block_params.into_iter().copied();
+    let mut block_params_iter = block_params.iter().copied();
     for param in abi_params {
         let val = ptr.offset_i64(fx, offset).store(
             fx,
@@ -260,7 +224,7 @@ pub(super) fn from_casted_value<'tcx>(
 
 /// Get a set of values to be passed as function arguments.
 pub(super) fn adjust_arg_for_abi<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     arg: CValue<'tcx>,
     arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
 ) -> SmallVec<[Value; 2]> {
@@ -283,9 +247,9 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
 /// Create a [`CValue`] containing the value of a function parameter adding clif function parameters
 /// as necessary.
 pub(super) fn cvalue_for_param<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    local: Option<mir::Local>,
+    local_field: Option<usize>,
     arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> Option<CValue<'tcx>> {
@@ -294,15 +258,11 @@ pub(super) fn cvalue_for_param<'tcx>(
         .into_iter()
         .map(|abi_param| {
             let block_param = block_params_iter.next().unwrap();
-            assert_eq!(
-                fx.bcx.func.dfg.value_type(block_param),
-                abi_param.value_type
-            );
+            assert_eq!(fx.bcx.func.dfg.value_type(block_param), abi_param.value_type);
             block_param
         })
         .collect::<SmallVec<[_; 2]>>();
 
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "arg",
@@ -321,29 +281,14 @@ pub(super) fn cvalue_for_param<'tcx>(
         }
         PassMode::Pair(_, _) => {
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
-            Some(CValue::by_val_pair(
-                block_params[0],
-                block_params[1],
-                arg_abi.layout,
-            ))
+            Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout))
         }
         PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)),
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => {
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
             assert_eq!(block_params.len(), 1, "{:?}", block_params);
-            Some(CValue::by_ref(
-                Pointer::new(block_params[0]),
-                arg_abi.layout,
-            ))
+            Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout))
         }
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: Some(_),
-            on_stack: _,
-        } => {
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
             Some(CValue::by_ref_unsized(
                 Pointer::new(block_params[0]),
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index a382963bf1e..e1c53224b4f 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -8,14 +8,13 @@ use smallvec::{smallvec, SmallVec};
 
 /// Can the given type be returned into an ssa var or does it need to be returned on the stack.
 pub(crate) fn can_return_to_ssa_var<'tcx>(
-    fx: &FunctionCx<'_, 'tcx, impl Module>,
+    fx: &FunctionCx<'_, '_, 'tcx>,
     func: &mir::Operand<'tcx>,
     args: &[mir::Operand<'tcx>],
 ) -> bool {
     let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
-    let fn_sig = fx
-        .tcx
-        .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
+    let fn_sig =
+        fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
 
     // Handle special calls like instrinsics and empty drop glue.
     let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
@@ -42,11 +41,7 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
     let fn_abi = if let Some(instance) = instance {
         FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
     } else {
-        FnAbi::of_fn_ptr(
-            &RevealAllLayoutCx(fx.tcx),
-            fn_ty.fn_sig(fx.tcx),
-            &extra_args,
-        )
+        FnAbi::of_fn_ptr(&RevealAllLayoutCx(fx.tcx), fn_ty.fn_sig(fx.tcx), &extra_args)
     };
     match fn_abi.ret.mode {
         PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => true,
@@ -58,15 +53,12 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
 /// Return a place where the return value of the current function can be written to. If necessary
 /// this adds an extra parameter pointing to where the return value needs to be stored.
 pub(super) fn codegen_return_param<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     ssa_analyzed: &rustc_index::vec::IndexVec<Local, crate::analyze::SsaKind>,
     block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> CPlace<'tcx> {
     let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
-        PassMode::Ignore => (
-            CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout),
-            smallvec![],
-        ),
+        PassMode::Ignore => (CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), smallvec![]),
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => {
             let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
             (
@@ -79,32 +71,19 @@ pub(super) fn codegen_return_param<'tcx>(
                 smallvec![],
             )
         }
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => {
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
             let ret_param = block_params_iter.next().unwrap();
             assert_eq!(fx.bcx.func.dfg.value_type(ret_param), pointer_ty(fx.tcx));
             (
-                CPlace::for_ptr(
-                    Pointer::new(ret_param),
-                    fx.fn_abi.as_ref().unwrap().ret.layout,
-                ),
+                CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.as_ref().unwrap().ret.layout),
                 smallvec![ret_param],
             )
         }
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: Some(_),
-            on_stack: _,
-        } => unreachable!("unsized return value"),
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
+            unreachable!("unsized return value")
+        }
     };
 
-    #[cfg(not(debug_assertions))]
-    let _ = ret_param;
-
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "ret",
@@ -120,27 +99,21 @@ pub(super) fn codegen_return_param<'tcx>(
 
 /// Invokes the closure with if necessary a value representing the return pointer. When the closure
 /// returns the call return value(s) if any are written to the correct place.
-pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+pub(super) fn codegen_with_call_return_arg<'tcx, T>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     ret_place: Option<CPlace<'tcx>>,
-    f: impl FnOnce(&mut FunctionCx<'_, 'tcx, M>, Option<Value>) -> (Inst, T),
+    f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> (Inst, T),
 ) -> (Inst, T) {
     let return_ptr = match ret_arg_abi.mode {
         PassMode::Ignore => None,
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => match ret_place {
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => match ret_place {
             Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)),
             None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot
         },
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: Some(_),
-            on_stack: _,
-        } => unreachable!("unsized return value"),
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
+            unreachable!("unsized return value")
+        }
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => None,
     };
 
@@ -169,7 +142,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
                 let results = fx
                     .bcx
                     .inst_results(call_inst)
-                    .into_iter()
+                    .iter()
                     .copied()
                     .collect::<SmallVec<[Value; 2]>>();
                 let result =
@@ -177,37 +150,24 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
                 ret_place.write_cvalue(fx, result);
             }
         }
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => {}
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: Some(_),
-            on_stack: _,
-        } => unreachable!("unsized return value"),
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {}
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
+            unreachable!("unsized return value")
+        }
     }
 
     (call_inst, meta)
 }
 
 /// Codegen a return instruction with the right return value(s) if any.
-pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) {
+pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
     match fx.fn_abi.as_ref().unwrap().ret.mode {
-        PassMode::Ignore
-        | PassMode::Indirect {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => {
+        PassMode::Ignore | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
             fx.bcx.ins().return_(&[]);
         }
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: Some(_),
-            on_stack: _,
-        } => unreachable!("unsized return value"),
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
+            unreachable!("unsized return value")
+        }
         PassMode::Direct(_) => {
             let place = fx.get_local_place(RETURN_PLACE);
             let ret_val = place.to_cvalue(fx).load_scalar(fx);
diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs
index 6c5916550ff..f60645a9f97 100644
--- a/compiler/rustc_codegen_cranelift/src/allocator.rs
+++ b/compiler/rustc_codegen_cranelift/src/allocator.rs
@@ -3,6 +3,7 @@
 
 use crate::prelude::*;
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 use rustc_span::symbol::sym;
 
@@ -66,13 +67,9 @@ fn codegen_inner(
         let callee_name = kind.fn_name(method.name);
         //eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns);
 
-        let func_id = module
-            .declare_function(&caller_name, Linkage::Export, &sig)
-            .unwrap();
+        let func_id = module.declare_function(&caller_name, Linkage::Export, &sig).unwrap();
 
-        let callee_func_id = module
-            .declare_function(&callee_name, Linkage::Import, &sig)
-            .unwrap();
+        let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
 
         let mut ctx = Context::new();
         ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig.clone());
@@ -96,11 +93,7 @@ fn codegen_inner(
             bcx.finalize();
         }
         module
-            .define_function(
-                func_id,
-                &mut ctx,
-                &mut cranelift_codegen::binemit::NullTrapSink {},
-            )
+            .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap();
         unwind_context.add_function(func_id, &ctx, module.isa());
     }
@@ -114,13 +107,10 @@ fn codegen_inner(
     let callee_name = kind.fn_name(sym::oom);
     //eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns);
 
-    let func_id = module
-        .declare_function("__rust_alloc_error_handler", Linkage::Export, &sig)
-        .unwrap();
+    let func_id =
+        module.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig).unwrap();
 
-    let callee_func_id = module
-        .declare_function(&callee_name, Linkage::Import, &sig)
-        .unwrap();
+    let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
 
     let mut ctx = Context::new();
     ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig);
@@ -143,11 +133,7 @@ fn codegen_inner(
         bcx.finalize();
     }
     module
-        .define_function(
-            func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
+        .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
         .unwrap();
     unwind_context.add_function(func_id, &ctx, module.isa());
 }
diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs
index 62fbcfe3f7a..efead25552f 100644
--- a/compiler/rustc_codegen_cranelift/src/analyze.rs
+++ b/compiler/rustc_codegen_cranelift/src/analyze.rs
@@ -11,7 +11,7 @@ pub(crate) enum SsaKind {
     Ssa,
 }
 
-pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, SsaKind> {
+pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
     let mut flag_map = fx
         .mir
         .local_decls
@@ -40,12 +40,7 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, S
         }
 
         match &bb.terminator().kind {
-            TerminatorKind::Call {
-                destination,
-                func,
-                args,
-                ..
-            } => {
+            TerminatorKind::Call { destination, func, args, .. } => {
                 if let Some((dest_place, _dest_bb)) = destination {
                     if !crate::abi::can_return_to_ssa_var(fx, func, args) {
                         not_ssa(&mut flag_map, dest_place.local)
diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs
index 96579054389..7583fc42407 100644
--- a/compiler/rustc_codegen_cranelift/src/archive.rs
+++ b/compiler/rustc_codegen_cranelift/src/archive.rs
@@ -12,10 +12,7 @@ use object::{Object, ObjectSymbol, SymbolKind};
 
 #[derive(Debug)]
 enum ArchiveEntry {
-    FromArchive {
-        archive_index: usize,
-        entry_index: usize,
-    },
+    FromArchive { archive_index: usize, entry_index: usize },
     File(PathBuf),
 }
 
@@ -30,7 +27,6 @@ pub(crate) struct ArArchiveBuilder<'a> {
     // Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at
     // the end of an archive for linkers to not get confused.
     entries: Vec<(String, ArchiveEntry)>,
-    update_symbols: bool,
 }
 
 impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
@@ -46,10 +42,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                 let entry = entry.unwrap();
                 entries.push((
                     String::from_utf8(entry.header().identifier().to_vec()).unwrap(),
-                    ArchiveEntry::FromArchive {
-                        archive_index: 0,
-                        entry_index: i,
-                    },
+                    ArchiveEntry::FromArchive { archive_index: 0, entry_index: i },
                 ));
                 i += 1;
             }
@@ -69,7 +62,6 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
 
             src_archives,
             entries,
-            update_symbols: false,
         }
     }
 
@@ -95,14 +87,9 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
 
     fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) {
         let location = find_library(name, &self.lib_search_paths, self.sess);
-        self.add_archive(location.clone(), |_| false)
-            .unwrap_or_else(|e| {
-                panic!(
-                    "failed to add native library {}: {}",
-                    location.to_string_lossy(),
-                    e
-                );
-            });
+        self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| {
+            panic!("failed to add native library {}: {}", location.to_string_lossy(), e);
+        });
     }
 
     fn add_rlib(
@@ -136,9 +123,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
         })
     }
 
-    fn update_symbols(&mut self) {
-        self.update_symbols = true;
-    }
+    fn update_symbols(&mut self) {}
 
     fn build(mut self) {
         enum BuilderKind {
@@ -156,10 +141,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             // FIXME only read the symbol table of the object files to avoid having to keep all
             // object files in memory at once, or read them twice.
             let data = match entry {
-                ArchiveEntry::FromArchive {
-                    archive_index,
-                    entry_index,
-                } => {
+                ArchiveEntry::FromArchive { archive_index, entry_index } => {
                     // FIXME read symbols from symtab
                     use std::io::Read;
                     let (ref _src_archive_path, ref mut src_archive) =
@@ -225,10 +207,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                             err
                         ));
                     }),
-                    entries
-                        .iter()
-                        .map(|(name, _)| name.as_bytes().to_vec())
-                        .collect(),
+                    entries.iter().map(|(name, _)| name.as_bytes().to_vec()).collect(),
                     ar::GnuSymbolTableFormat::Size32,
                     symbol_table,
                 )
@@ -271,8 +250,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                 .expect("Couldn't run ranlib");
 
             if !status.success() {
-                self.sess
-                    .fatal(&format!("Ranlib exited with code {:?}", status.code()));
+                self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
             }
         }
     }
@@ -292,13 +270,8 @@ impl<'a> ArArchiveBuilder<'a> {
             let file_name = String::from_utf8(entry.header().identifier().to_vec())
                 .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
             if !skip(&file_name) {
-                self.entries.push((
-                    file_name,
-                    ArchiveEntry::FromArchive {
-                        archive_index,
-                        entry_index: i,
-                    },
-                ));
+                self.entries
+                    .push((file_name, ArchiveEntry::FromArchive { archive_index, entry_index: i }));
             }
             i += 1;
         }
diff --git a/compiler/rustc_codegen_cranelift/src/atomic_shim.rs b/compiler/rustc_codegen_cranelift/src/atomic_shim.rs
deleted file mode 100644
index 674e6d90751..00000000000
--- a/compiler/rustc_codegen_cranelift/src/atomic_shim.rs
+++ /dev/null
@@ -1,185 +0,0 @@
-//! Atomic intrinsics are implemented using a global lock for now, as Cranelift doesn't support
-//! atomic operations yet.
-
-// FIXME implement atomic instructions in Cranelift.
-
-use crate::prelude::*;
-
-#[cfg(all(feature = "jit", unix))]
-#[no_mangle]
-static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER;
-
-pub(crate) fn init_global_lock(
-    module: &mut impl Module,
-    bcx: &mut FunctionBuilder<'_>,
-    use_jit: bool,
-) {
-    if use_jit {
-        // When using JIT, dylibs won't find the __cg_clif_global_atomic_mutex data object defined here,
-        // so instead we define it in the cg_clif dylib.
-
-        return;
-    }
-
-    let mut data_ctx = DataContext::new();
-    data_ctx.define_zeroinit(1024); // 1024 bytes should be big enough on all platforms.
-    data_ctx.set_align(16);
-    let atomic_mutex = module
-        .declare_data(
-            "__cg_clif_global_atomic_mutex",
-            Linkage::Export,
-            true,
-            false,
-        )
-        .unwrap();
-    module.define_data(atomic_mutex, &data_ctx).unwrap();
-
-    let pthread_mutex_init = module
-        .declare_function(
-            "pthread_mutex_init",
-            Linkage::Import,
-            &cranelift_codegen::ir::Signature {
-                call_conv: module.target_config().default_call_conv,
-                params: vec![
-                    AbiParam::new(
-                        module.target_config().pointer_type(), /* *mut pthread_mutex_t */
-                    ),
-                    AbiParam::new(
-                        module.target_config().pointer_type(), /* *const pthread_mutex_attr_t */
-                    ),
-                ],
-                returns: vec![AbiParam::new(types::I32 /* c_int */)],
-            },
-        )
-        .unwrap();
-
-    let pthread_mutex_init = module.declare_func_in_func(pthread_mutex_init, bcx.func);
-
-    let atomic_mutex = module.declare_data_in_func(atomic_mutex, bcx.func);
-    let atomic_mutex = bcx
-        .ins()
-        .global_value(module.target_config().pointer_type(), atomic_mutex);
-
-    let nullptr = bcx.ins().iconst(module.target_config().pointer_type(), 0);
-
-    bcx.ins().call(pthread_mutex_init, &[atomic_mutex, nullptr]);
-}
-
-pub(crate) fn init_global_lock_constructor(
-    module: &mut impl Module,
-    constructor_name: &str,
-) -> FuncId {
-    let sig = Signature::new(CallConv::SystemV);
-    let init_func_id = module
-        .declare_function(constructor_name, Linkage::Export, &sig)
-        .unwrap();
-
-    let mut ctx = Context::new();
-    ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig);
-    {
-        let mut func_ctx = FunctionBuilderContext::new();
-        let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
-
-        let block = bcx.create_block();
-        bcx.switch_to_block(block);
-
-        crate::atomic_shim::init_global_lock(module, &mut bcx, false);
-
-        bcx.ins().return_(&[]);
-        bcx.seal_all_blocks();
-        bcx.finalize();
-    }
-    module
-        .define_function(
-            init_func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
-        .unwrap();
-
-    init_func_id
-}
-
-pub(crate) fn lock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    let atomic_mutex = fx
-        .cx
-        .module
-        .declare_data(
-            "__cg_clif_global_atomic_mutex",
-            Linkage::Import,
-            true,
-            false,
-        )
-        .unwrap();
-
-    let pthread_mutex_lock = fx
-        .cx
-        .module
-        .declare_function(
-            "pthread_mutex_lock",
-            Linkage::Import,
-            &cranelift_codegen::ir::Signature {
-                call_conv: fx.cx.module.target_config().default_call_conv,
-                params: vec![AbiParam::new(
-                    fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */
-                )],
-                returns: vec![AbiParam::new(types::I32 /* c_int */)],
-            },
-        )
-        .unwrap();
-
-    let pthread_mutex_lock = fx
-        .cx
-        .module
-        .declare_func_in_func(pthread_mutex_lock, fx.bcx.func);
-
-    let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
-    let atomic_mutex = fx
-        .bcx
-        .ins()
-        .global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex);
-
-    fx.bcx.ins().call(pthread_mutex_lock, &[atomic_mutex]);
-}
-
-pub(crate) fn unlock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    let atomic_mutex = fx
-        .cx
-        .module
-        .declare_data(
-            "__cg_clif_global_atomic_mutex",
-            Linkage::Import,
-            true,
-            false,
-        )
-        .unwrap();
-
-    let pthread_mutex_unlock = fx
-        .cx
-        .module
-        .declare_function(
-            "pthread_mutex_unlock",
-            Linkage::Import,
-            &cranelift_codegen::ir::Signature {
-                call_conv: fx.cx.module.target_config().default_call_conv,
-                params: vec![AbiParam::new(
-                    fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */
-                )],
-                returns: vec![AbiParam::new(types::I32 /* c_int */)],
-            },
-        )
-        .unwrap();
-
-    let pthread_mutex_unlock = fx
-        .cx
-        .module
-        .declare_func_in_func(pthread_mutex_unlock, fx.bcx.func);
-
-    let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
-    let atomic_mutex = fx
-        .bcx
-        .ins()
-        .global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex);
-
-    fx.bcx.ins().call(pthread_mutex_unlock, &[atomic_mutex]);
-}
diff --git a/compiler/rustc_codegen_cranelift/src/backend.rs b/compiler/rustc_codegen_cranelift/src/backend.rs
index 0ce34c904bd..eb7927fc4ad 100644
--- a/compiler/rustc_codegen_cranelift/src/backend.rs
+++ b/compiler/rustc_codegen_cranelift/src/backend.rs
@@ -8,7 +8,7 @@ use rustc_session::Session;
 use cranelift_module::FuncId;
 
 use object::write::*;
-use object::{RelocationEncoding, RelocationKind, SectionKind, SymbolFlags};
+use object::{RelocationEncoding, SectionKind, SymbolFlags};
 
 use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
 
@@ -22,9 +22,7 @@ pub(crate) trait WriteMetadata {
 
 impl WriteMetadata for object::write::Object {
     fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, _is_like_osx: bool) {
-        let segment = self
-            .segment_name(object::write::StandardSegment::Data)
-            .to_vec();
+        let segment = self.segment_name(object::write::StandardSegment::Data).to_vec();
         let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data);
         let offset = self.append_section_data(section_id, &data, 1);
         // For MachO and probably PE this is necessary to prevent the linker from throwing away the
@@ -74,11 +72,7 @@ impl WriteDebugInfo for ObjectProduct {
         let section_id = self.object.add_section(
             segment,
             name,
-            if id == SectionId::EhFrame {
-                SectionKind::ReadOnlyData
-            } else {
-                SectionKind::Debug
-            },
+            if id == SectionId::EhFrame { SectionKind::ReadOnlyData } else { SectionKind::Debug },
         );
         self.object
             .section_mut(section_id)
@@ -118,49 +112,6 @@ impl WriteDebugInfo for ObjectProduct {
     }
 }
 
-// FIXME remove once atomic instructions are implemented in Cranelift.
-pub(crate) trait AddConstructor {
-    fn add_constructor(&mut self, func_id: FuncId);
-}
-
-impl AddConstructor for ObjectProduct {
-    fn add_constructor(&mut self, func_id: FuncId) {
-        let symbol = self.function_symbol(func_id);
-        let segment = self
-            .object
-            .segment_name(object::write::StandardSegment::Data);
-        let init_array_section =
-            self.object
-                .add_section(segment.to_vec(), b".init_array".to_vec(), SectionKind::Data);
-        let address_size = self
-            .object
-            .architecture()
-            .address_size()
-            .expect("address_size must be known")
-            .bytes();
-        self.object.append_section_data(
-            init_array_section,
-            &std::iter::repeat(0)
-                .take(address_size.into())
-                .collect::<Vec<u8>>(),
-            8,
-        );
-        self.object
-            .add_relocation(
-                init_array_section,
-                object::write::Relocation {
-                    offset: 0,
-                    size: address_size * 8,
-                    kind: RelocationKind::Absolute,
-                    encoding: RelocationEncoding::Generic,
-                    symbol,
-                    addend: 0,
-                },
-            )
-            .unwrap();
-    }
-}
-
 pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
     let triple = crate::build_isa(sess).triple().clone();
 
@@ -175,10 +126,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object
         target_lexicon::Architecture::X86_64 => object::Architecture::X86_64,
         target_lexicon::Architecture::Arm(_) => object::Architecture::Arm,
         target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64,
-        architecture => sess.fatal(&format!(
-            "target architecture {:?} is unsupported",
-            architecture,
-        )),
+        architecture => {
+            sess.fatal(&format!("target architecture {:?} is unsupported", architecture,))
+        }
     };
     let endian = match triple.endianness().unwrap() {
         target_lexicon::Endianness::Little => object::Endianness::Little,
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 4842628a99d..b34a29c25b9 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -1,5 +1,6 @@
 //! Codegen of a single function
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiExt;
@@ -7,11 +8,7 @@ use rustc_target::abi::call::FnAbi;
 
 use crate::prelude::*;
 
-pub(crate) fn codegen_fn<'tcx>(
-    cx: &mut crate::CodegenCx<'tcx, impl Module>,
-    instance: Instance<'tcx>,
-    linkage: Linkage,
-) {
+pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let _inst_guard =
@@ -23,7 +20,7 @@ pub(crate) fn codegen_fn<'tcx>(
     // Declare function
     let name = tcx.symbol_name(instance).name.to_string();
     let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
-    let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
+    let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();
 
     cx.cached_context.clear();
 
@@ -38,9 +35,8 @@ pub(crate) fn codegen_fn<'tcx>(
 
     // Predefine blocks
     let start_block = bcx.create_block();
-    let block_map: IndexVec<BasicBlock, Block> = (0..mir.basic_blocks().len())
-        .map(|_| bcx.create_block())
-        .collect();
+    let block_map: IndexVec<BasicBlock, Block> =
+        (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
 
     // Make FunctionCx
     let pointer_type = cx.module.target_config().pointer_type();
@@ -68,22 +64,23 @@ pub(crate) fn codegen_fn<'tcx>(
         inline_asm_index: 0,
     };
 
-    let arg_uninhabited = fx.mir.args_iter().any(|arg| {
-        fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty))
-            .abi
-            .is_uninhabited()
-    });
+    let arg_uninhabited = fx
+        .mir
+        .args_iter()
+        .any(|arg| fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
 
-    if arg_uninhabited {
-        fx.bcx
-            .append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+    if !crate::constant::check_constants(&mut fx) {
+        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+        fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+        crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted");
+    } else if arg_uninhabited {
+        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
         fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
         crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument");
     } else {
         tcx.sess.time("codegen clif ir", || {
-            tcx.sess.time("codegen prelude", || {
-                crate::abi::codegen_fn_prelude(&mut fx, start_block)
-            });
+            tcx.sess
+                .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
             codegen_fn_content(&mut fx);
         });
     }
@@ -131,11 +128,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let module = &mut cx.module;
     tcx.sess.time("define function", || {
         module
-            .define_function(
-                func_id,
-                context,
-                &mut cranelift_codegen::binemit::NullTrapSink {},
-            )
+            .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap()
     });
 
@@ -149,14 +142,12 @@ pub(crate) fn codegen_fn<'tcx>(
         &clif_comments,
     );
 
-    if let Some(mach_compile_result) = &context.mach_compile_result {
-        if let Some(disasm) = &mach_compile_result.disasm {
-            crate::pretty_clif::write_ir_file(
-                tcx,
-                &format!("{}.vcode", tcx.symbol_name(instance).name),
-                |file| file.write_all(disasm.as_bytes()),
-            )
-        }
+    if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
+        crate::pretty_clif::write_ir_file(
+            tcx,
+            &format!("{}.vcode", tcx.symbol_name(instance).name),
+            |file| file.write_all(disasm.as_bytes()),
+        )
     }
 
     // Define debuginfo for function
@@ -199,16 +190,13 @@ pub(crate) fn verify_func(
                     Some(Box::new(writer)),
                     err,
                 );
-                tcx.sess
-                    .fatal(&format!("cranelift verify error:\n{}", pretty_error));
+                tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error));
             }
         }
     });
 }
 
-fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    crate::constant::check_constants(fx);
-
+fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
     for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
         let block = fx.get_block(bb);
         fx.bcx.switch_to_block(block);
@@ -228,14 +216,9 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
             codegen_stmt(fx, block, stmt);
         }
 
-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
             let mut terminator_head = "\n".to_string();
-            bb_data
-                .terminator()
-                .kind
-                .fmt_head(&mut terminator_head)
-                .unwrap();
+            bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
             let inst = fx.bcx.func.layout.last_inst(block).unwrap();
             fx.add_comment(inst, terminator_head);
         }
@@ -267,13 +250,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
             TerminatorKind::Return => {
                 crate::abi::codegen_return(fx);
             }
-            TerminatorKind::Assert {
-                cond,
-                expected,
-                msg,
-                target,
-                cleanup: _,
-            } => {
+            TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => {
                 if !fx.tcx.sess.overflow_checks() {
                     if let mir::AssertKind::OverflowNeg(_) = *msg {
                         let target = fx.get_block(*target);
@@ -319,11 +296,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
                 }
             }
 
-            TerminatorKind::SwitchInt {
-                discr,
-                switch_ty,
-                targets,
-            } => {
+            TerminatorKind::SwitchInt { discr, switch_ty, targets } => {
                 let discr = codegen_operand(fx, discr).load_scalar(fx);
 
                 let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind()
@@ -433,11 +406,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
             | TerminatorKind::GeneratorDrop => {
                 bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
             }
-            TerminatorKind::Drop {
-                place,
-                target,
-                unwind: _,
-            } => {
+            TerminatorKind::Drop { place, target, unwind: _ } => {
                 let drop_place = codegen_place(fx, *place);
                 crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place);
 
@@ -452,7 +421,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
 }
 
 fn codegen_stmt<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     #[allow(unused_variables)] cur_block: Block,
     stmt: &Statement<'tcx>,
 ) {
@@ -460,20 +429,19 @@ fn codegen_stmt<'tcx>(
 
     fx.set_debug_loc(stmt.source_info);
 
-    #[cfg(false_debug_assertions)]
+    #[cfg(disabled)]
     match &stmt.kind {
         StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
         _ => {
-            let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
-            fx.add_comment(inst, format!("{:?}", stmt));
+            if fx.clif_comments.enabled() {
+                let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+                fx.add_comment(inst, format!("{:?}", stmt));
+            }
         }
     }
 
     match &stmt.kind {
-        StatementKind::SetDiscriminant {
-            place,
-            variant_index,
-        } => {
+        StatementKind::SetDiscriminant { place, variant_index } => {
             let place = codegen_place(fx, **place);
             crate::discriminant::codegen_set_discriminant(fx, place, *variant_index);
         }
@@ -494,16 +462,16 @@ fn codegen_stmt<'tcx>(
                     let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
                     lval.write_cvalue(fx, val);
                 }
-                Rvalue::BinaryOp(bin_op, ref lhs, ref rhs) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
                     lval.write_cvalue(fx, res);
                 }
-                Rvalue::CheckedBinaryOp(bin_op, ref lhs, ref rhs) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = if !fx.tcx.sess.overflow_checks() {
                         let val =
@@ -594,19 +562,11 @@ fn codegen_stmt<'tcx>(
                     let from_ty = operand.layout().ty;
                     let to_ty = fx.monomorphize(to_ty);
 
-                    fn is_fat_ptr<'tcx>(
-                        fx: &FunctionCx<'_, 'tcx, impl Module>,
-                        ty: Ty<'tcx>,
-                    ) -> bool {
+                    fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
                         ty.builtin_deref(true)
-                            .map(
-                                |ty::TypeAndMut {
-                                     ty: pointee_ty,
-                                     mutbl: _,
-                                 }| {
-                                    has_ptr_meta(fx.tcx, pointee_ty)
-                                },
-                            )
+                            .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| {
+                                has_ptr_meta(fx.tcx, pointee_ty)
+                            })
                             .unwrap_or(false)
                     }
 
@@ -626,50 +586,22 @@ fn codegen_stmt<'tcx>(
                             ty::Uint(_) | ty::Int(_) => {}
                             _ => unreachable!("cast adt {} -> {}", from_ty, to_ty),
                         }
+                        let to_clif_ty = fx.clif_type(to_ty).unwrap();
 
-                        use rustc_target::abi::{Int, TagEncoding, Variants};
-
-                        match operand.layout().variants {
-                            Variants::Single { index } => {
-                                let discr = operand
-                                    .layout()
-                                    .ty
-                                    .discriminant_for_variant(fx.tcx, index)
-                                    .unwrap();
-                                let discr = if discr.ty.is_signed() {
-                                    fx.layout_of(discr.ty).size.sign_extend(discr.val)
-                                } else {
-                                    discr.val
-                                };
-                                let discr = discr.into();
-
-                                let discr = CValue::const_val(fx, fx.layout_of(to_ty), discr);
-                                lval.write_cvalue(fx, discr);
-                            }
-                            Variants::Multiple {
-                                ref tag,
-                                tag_field,
-                                tag_encoding: TagEncoding::Direct,
-                                variants: _,
-                            } => {
-                                let cast_to = fx.clif_type(dest_layout.ty).unwrap();
-
-                                // Read the tag/niche-encoded discriminant from memory.
-                                let encoded_discr =
-                                    operand.value_field(fx, mir::Field::new(tag_field));
-                                let encoded_discr = encoded_discr.load_scalar(fx);
-
-                                // Decode the discriminant (specifically if it's niche-encoded).
-                                let signed = match tag.value {
-                                    Int(_, signed) => signed,
-                                    _ => false,
-                                };
-                                let val = clif_intcast(fx, encoded_discr, cast_to, signed);
-                                let val = CValue::by_val(val, dest_layout);
-                                lval.write_cvalue(fx, val);
-                            }
-                            Variants::Multiple { .. } => unreachable!(),
-                        }
+                        let discriminant = crate::discriminant::codegen_get_discriminant(
+                            fx,
+                            operand,
+                            fx.layout_of(operand.layout().ty.discriminant_ty(fx.tcx)),
+                        )
+                        .load_scalar(fx);
+
+                        let res = crate::cast::clif_intcast(
+                            fx,
+                            discriminant,
+                            to_clif_ty,
+                            to_ty.is_signed(),
+                        );
+                        lval.write_cvalue(fx, CValue::by_val(res, dest_layout));
                     } else {
                         let to_clif_ty = fx.clif_type(to_ty).unwrap();
                         let from = operand.load_scalar(fx);
@@ -725,13 +657,14 @@ fn codegen_stmt<'tcx>(
                         .val
                         .try_to_bits(fx.tcx.data_layout.pointer_size)
                         .unwrap();
-                    if fx.clif_type(operand.layout().ty) == Some(types::I8) {
+                    if operand.layout().size.bytes() == 0 {
+                        // Do nothing for ZST's
+                    } else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
                         let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
                         // FIXME use emit_small_memset where possible
                         let addr = lval.to_ptr().get_addr(fx);
                         let val = operand.load_scalar(fx);
-                        fx.bcx
-                            .call_memset(fx.cx.module.target_config(), addr, val, times);
+                        fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times);
                     } else {
                         let loop_block = fx.bcx.create_block();
                         let loop_block2 = fx.bcx.create_block();
@@ -766,25 +699,19 @@ fn codegen_stmt<'tcx>(
                     let content_ty = fx.monomorphize(content_ty);
                     let layout = fx.layout_of(content_ty);
                     let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64);
-                    let llalign = fx
-                        .bcx
-                        .ins()
-                        .iconst(usize_type, layout.align.abi.bytes() as i64);
+                    let llalign = fx.bcx.ins().iconst(usize_type, layout.align.abi.bytes() as i64);
                     let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty));
 
                     // Allocate space:
-                    let def_id = match fx
-                        .tcx
-                        .lang_items()
-                        .require(rustc_hir::LangItem::ExchangeMalloc)
-                    {
-                        Ok(id) => id,
-                        Err(s) => {
-                            fx.tcx
-                                .sess
-                                .fatal(&format!("allocation of `{}` {}", box_layout.ty, s));
-                        }
-                    };
+                    let def_id =
+                        match fx.tcx.lang_items().require(rustc_hir::LangItem::ExchangeMalloc) {
+                            Ok(id) => id,
+                            Err(s) => {
+                                fx.tcx
+                                    .sess
+                                    .fatal(&format!("allocation of `{}` {}", box_layout.ty, s));
+                            }
+                        };
                     let instance = ty::Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
                     let func_ref = fx.get_function_ref(instance);
                     let call = fx.bcx.ins().call(func_ref, &[llsize, llalign]);
@@ -792,10 +719,11 @@ fn codegen_stmt<'tcx>(
                     lval.write_cvalue(fx, CValue::by_val(ptr, box_layout));
                 }
                 Rvalue::NullaryOp(NullOp::SizeOf, ty) => {
-                    assert!(lval
-                        .layout()
-                        .ty
-                        .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all()));
+                    assert!(
+                        lval.layout()
+                            .ty
+                            .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all())
+                    );
                     let ty_size = fx.layout_of(fx.monomorphize(ty)).size.bytes();
                     let val =
                         CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), ty_size.into());
@@ -823,11 +751,7 @@ fn codegen_stmt<'tcx>(
 
         StatementKind::LlvmInlineAsm(asm) => {
             use rustc_span::symbol::Symbol;
-            let LlvmInlineAsm {
-                asm,
-                outputs,
-                inputs,
-            } = &**asm;
+            let LlvmInlineAsm { asm, outputs, inputs } = &**asm;
             let rustc_hir::LlvmInlineAsmInner {
                 asm: asm_code,         // Name
                 outputs: output_names, // Vec<LlvmInlineAsmOutput>
@@ -843,15 +767,9 @@ fn codegen_stmt<'tcx>(
                     // Black box
                 }
                 "mov %rbx, %rsi\n                  cpuid\n                  xchg %rbx, %rsi" => {
-                    assert_eq!(
-                        input_names,
-                        &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]
-                    );
+                    assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]);
                     assert_eq!(output_names.len(), 4);
-                    for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"])
-                        .iter()
-                        .enumerate()
-                    {
+                    for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() {
                         assert_eq!(&output_names[i].constraint.as_str(), c);
                         assert!(!output_names[i].is_rw);
                         assert!(!output_names[i].is_indirect);
@@ -897,12 +815,7 @@ fn codegen_stmt<'tcx>(
                     crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported");
                 }
                 // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
-                _ if fx
-                    .tcx
-                    .symbol_name(fx.instance)
-                    .name
-                    .starts_with("___chkstk") =>
-                {
+                _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => {
                     crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
                 }
                 _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => {
@@ -919,30 +832,38 @@ fn codegen_stmt<'tcx>(
             }
         }
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
+        StatementKind::CopyNonOverlapping(inner) => {
+            let dst = codegen_operand(fx, &inner.dst);
+            let pointee = dst
+                .layout()
+                .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+                .expect("Expected pointer");
+            let dst = dst.load_scalar(fx);
+            let src = codegen_operand(fx, &inner.src).load_scalar(fx);
+            let count = codegen_operand(fx, &inner.count).load_scalar(fx);
+            let elem_size: u64 = pointee.size.bytes();
+            let bytes =
+                if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
+            fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
+        }
     }
 }
 
-fn codegen_array_len<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    place: CPlace<'tcx>,
-) -> Value {
+fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
     match *place.layout().ty.kind() {
         ty::Array(_elem_ty, len) => {
-            let len = fx
-                .monomorphize(len)
-                .eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
+            let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
             fx.bcx.ins().iconst(fx.pointer_type, len)
         }
-        ty::Slice(_elem_ty) => place
-            .to_ptr_maybe_unsized()
-            .1
-            .expect("Length metadata for slice place"),
+        ty::Slice(_elem_ty) => {
+            place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place")
+        }
         _ => bug!("Rvalue::Len({:?})", place),
     }
 }
 
 pub(crate) fn codegen_place<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     place: Place<'tcx>,
 ) -> CPlace<'tcx> {
     let mut cplace = fx.get_local_place(place.local);
@@ -959,11 +880,7 @@ pub(crate) fn codegen_place<'tcx>(
                 let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx);
                 cplace = cplace.place_index(fx, index);
             }
-            PlaceElem::ConstantIndex {
-                offset,
-                min_length: _,
-                from_end,
-            } => {
+            PlaceElem::ConstantIndex { offset, min_length: _, from_end } => {
                 let offset: u64 = offset;
                 let index = if !from_end {
                     fx.bcx.ins().iconst(fx.pointer_type, offset as i64)
@@ -1014,7 +931,7 @@ pub(crate) fn codegen_place<'tcx>(
 }
 
 pub(crate) fn codegen_operand<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     operand: &Operand<'tcx>,
 ) -> CValue<'tcx> {
     match operand {
@@ -1026,34 +943,24 @@ pub(crate) fn codegen_operand<'tcx>(
     }
 }
 
-pub(crate) fn codegen_panic<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    msg_str: &str,
-    span: Span,
-) {
+pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) {
     let location = fx.get_caller_location(span).load_scalar(fx);
 
     let msg_ptr = fx.anonymous_str("assert", msg_str);
-    let msg_len = fx
-        .bcx
-        .ins()
-        .iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
+    let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
     let args = [msg_ptr, msg_len, location];
 
     codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span);
 }
 
 pub(crate) fn codegen_panic_inner<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     lang_item: rustc_hir::LangItem,
     args: &[Value],
     span: Span,
 ) {
-    let def_id = fx
-        .tcx
-        .lang_items()
-        .require(lang_item)
-        .unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
+    let def_id =
+        fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
 
     let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
     let symbol_name = fx.tcx.symbol_name(instance).name;
diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
index be369b07fdd..983839d48d2 100644
--- a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
@@ -27,13 +27,7 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
         config.opts.cg.panic = Some(PanicStrategy::Abort);
         config.opts.debugging_opts.panic_abort_tests = true;
         config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| {
-            std::env::current_exe()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .to_owned()
+            std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned()
         }));
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs
index 83e5dc6e672..e7cd5edbbf6 100644
--- a/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs
+++ b/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs
@@ -46,15 +46,8 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
 
         config.opts.cg.panic = Some(PanicStrategy::Abort);
         config.opts.debugging_opts.panic_abort_tests = true;
-        config.opts.maybe_sysroot = Some(
-            std::env::current_exe()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .to_owned(),
-        );
+        config.opts.maybe_sysroot =
+            Some(std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned());
     }
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/cast.rs b/compiler/rustc_codegen_cranelift/src/cast.rs
index 57204de1135..74c5e09f08d 100644
--- a/compiler/rustc_codegen_cranelift/src/cast.rs
+++ b/compiler/rustc_codegen_cranelift/src/cast.rs
@@ -3,7 +3,7 @@
 use crate::prelude::*;
 
 pub(crate) fn clif_intcast(
-    fx: &mut FunctionCx<'_, '_, impl Module>,
+    fx: &mut FunctionCx<'_, '_, '_>,
     val: Value,
     to: Type,
     signed: bool,
@@ -40,18 +40,14 @@ pub(crate) fn clif_intcast(
         // reduce
         (types::I128, _) => {
             let (lsb, _msb) = fx.bcx.ins().isplit(val);
-            if to == types::I64 {
-                lsb
-            } else {
-                fx.bcx.ins().ireduce(to, lsb)
-            }
+            if to == types::I64 { lsb } else { fx.bcx.ins().ireduce(to, lsb) }
         }
         (_, _) => fx.bcx.ins().ireduce(to, val),
     }
 }
 
 pub(crate) fn clif_int_or_float_cast(
-    fx: &mut FunctionCx<'_, '_, impl Module>,
+    fx: &mut FunctionCx<'_, '_, '_>,
     from: Value,
     from_signed: bool,
     to_ty: Type,
@@ -87,11 +83,7 @@ pub(crate) fn clif_int_or_float_cast(
                 },
             );
 
-            let from_rust_ty = if from_signed {
-                fx.tcx.types.i128
-            } else {
-                fx.tcx.types.u128
-            };
+            let from_rust_ty = if from_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
 
             let to_rust_ty = match to_ty {
                 types::F32 => fx.tcx.types.f32,
@@ -100,11 +92,7 @@ pub(crate) fn clif_int_or_float_cast(
             };
 
             return fx
-                .easy_call(
-                    &name,
-                    &[CValue::by_val(from, fx.layout_of(from_rust_ty))],
-                    to_rust_ty,
-                )
+                .easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
                 .load_scalar(fx);
         }
 
@@ -138,18 +126,10 @@ pub(crate) fn clif_int_or_float_cast(
                 _ => unreachable!(),
             };
 
-            let to_rust_ty = if to_signed {
-                fx.tcx.types.i128
-            } else {
-                fx.tcx.types.u128
-            };
+            let to_rust_ty = if to_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
 
             return fx
-                .easy_call(
-                    &name,
-                    &[CValue::by_val(from, fx.layout_of(from_rust_ty))],
-                    to_rust_ty,
-                )
+                .easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
                 .load_scalar(fx);
         }
 
diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
index 866ba90e4ae..ffe1922ab90 100644
--- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
+++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
@@ -5,13 +5,17 @@ use cranelift_codegen::ir::ArgumentPurpose;
 use crate::prelude::*;
 
 pub(crate) fn maybe_codegen<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     checked: bool,
     lhs: CValue<'tcx>,
     rhs: CValue<'tcx>,
 ) -> Option<CValue<'tcx>> {
-    if lhs.layout().ty != fx.tcx.types.u128 && lhs.layout().ty != fx.tcx.types.i128 {
+    if lhs.layout().ty != fx.tcx.types.u128
+        && lhs.layout().ty != fx.tcx.types.i128
+        && rhs.layout().ty != fx.tcx.types.u128
+        && rhs.layout().ty != fx.tcx.types.i128
+    {
         return None;
     }
 
@@ -27,27 +31,57 @@ pub(crate) fn maybe_codegen<'tcx>(
         }
         BinOp::Add | BinOp::Sub if !checked => None,
         BinOp::Mul if !checked => {
-            let val_ty = if is_signed {
-                fx.tcx.types.i128
+            let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
+            if fx.tcx.sess.target.is_like_windows {
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args =
+                    [ret_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                fx.lib_call(
+                    "__multi3",
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    vec![],
+                    &args,
+                );
+                Some(ret_place.to_cvalue(fx))
             } else {
-                fx.tcx.types.u128
-            };
-            Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+                Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            }
         }
         BinOp::Add | BinOp::Sub | BinOp::Mul => {
             assert!(checked);
             let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
             let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-            let param_types = vec![
-                AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
-                AbiParam::new(types::I128),
-                AbiParam::new(types::I128),
-            ];
-            let args = [
-                out_place.to_ptr().get_addr(fx),
-                lhs.load_scalar(fx),
-                rhs.load_scalar(fx),
-            ];
+            let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
+                )
+            } else {
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(types::I128),
+                        AbiParam::new(types::I128),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
+                )
+            };
             let name = match (bin_op, is_signed) {
                 (BinOp::Add, false) => "__rust_u128_addo",
                 (BinOp::Add, true) => "__rust_i128_addo",
@@ -61,20 +95,33 @@ pub(crate) fn maybe_codegen<'tcx>(
             Some(out_place.to_cvalue(fx))
         }
         BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
-        BinOp::Div => {
-            assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128))
-            } else {
-                Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128))
-            }
-        }
-        BinOp::Rem => {
+        BinOp::Div | BinOp::Rem => {
             assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128))
+            let name = match (bin_op, is_signed) {
+                (BinOp::Div, false) => "__udivti3",
+                (BinOp::Div, true) => "__divti3",
+                (BinOp::Rem, false) => "__umodti3",
+                (BinOp::Rem, true) => "__modti3",
+                _ => unreachable!(),
+            };
+            if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                let ret = fx.lib_call(
+                    name,
+                    vec![AbiParam::new(pointer_ty(fx.tcx)), AbiParam::new(pointer_ty(fx.tcx))],
+                    vec![AbiParam::new(types::I64X2)],
+                    &args,
+                )[0];
+                // FIXME use bitcast instead of store to get from i64x2 to i128
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
+                Some(ret_place.to_cvalue(fx))
             } else {
-                Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128))
+                Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
             }
         }
         BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {
@@ -97,70 +144,23 @@ pub(crate) fn maybe_codegen<'tcx>(
                 None
             };
 
-            // Optimize `val >> 64`, because compiler_builtins uses it to deconstruct an 128bit
-            // integer into its lsb and msb.
-            // https://github.com/rust-lang-nursery/compiler-builtins/blob/79a6a1603d5672cbb9187ff41ff4d9b5048ac1cb/src/int/mod.rs#L217
-            if resolve_value_imm(fx.bcx.func, rhs_val) == Some(64) {
-                let (lhs_lsb, lhs_msb) = fx.bcx.ins().isplit(lhs_val);
-                let all_zeros = fx.bcx.ins().iconst(types::I64, 0);
-                let val = match (bin_op, is_signed) {
-                    (BinOp::Shr, false) => {
-                        let val = fx.bcx.ins().iconcat(lhs_msb, all_zeros);
-                        Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.u128)))
-                    }
-                    (BinOp::Shr, true) => {
-                        let sign = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, lhs_msb, 0);
-                        let all_ones = fx.bcx.ins().iconst(types::I64, u64::MAX as i64);
-                        let all_sign_bits = fx.bcx.ins().select(sign, all_zeros, all_ones);
-
-                        let val = fx.bcx.ins().iconcat(lhs_msb, all_sign_bits);
-                        Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.i128)))
-                    }
-                    (BinOp::Shl, _) => {
-                        let val_ty = if is_signed {
-                            fx.tcx.types.i128
-                        } else {
-                            fx.tcx.types.u128
-                        };
-                        let val = fx.bcx.ins().iconcat(all_zeros, lhs_lsb);
-                        Some(CValue::by_val(val, fx.layout_of(val_ty)))
-                    }
-                    _ => None,
-                };
-                if let Some(val) = val {
-                    if let Some(is_overflow) = is_overflow {
-                        let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
-                        let val = val.load_scalar(fx);
-                        return Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty)));
+            let truncated_rhs = clif_intcast(fx, rhs_val, types::I32, false);
+            let val = match bin_op {
+                BinOp::Shl => fx.bcx.ins().ishl(lhs_val, truncated_rhs),
+                BinOp::Shr => {
+                    if is_signed {
+                        fx.bcx.ins().sshr(lhs_val, truncated_rhs)
                     } else {
-                        return Some(val);
+                        fx.bcx.ins().ushr(lhs_val, truncated_rhs)
                     }
                 }
-            }
-
-            let truncated_rhs = clif_intcast(fx, rhs_val, types::I32, false);
-            let truncated_rhs = CValue::by_val(truncated_rhs, fx.layout_of(fx.tcx.types.u32));
-            let val = match (bin_op, is_signed) {
-                (BinOp::Shl, false) => {
-                    fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.u128)
-                }
-                (BinOp::Shl, true) => {
-                    fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.i128)
-                }
-                (BinOp::Shr, false) => {
-                    fx.easy_call("__lshrti3", &[lhs, truncated_rhs], fx.tcx.types.u128)
-                }
-                (BinOp::Shr, true) => {
-                    fx.easy_call("__ashrti3", &[lhs, truncated_rhs], fx.tcx.types.i128)
-                }
-                (_, _) => unreachable!(),
+                _ => unreachable!(),
             };
             if let Some(is_overflow) = is_overflow {
                 let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
-                let val = val.load_scalar(fx);
                 Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty)))
             } else {
-                Some(val)
+                Some(CValue::by_val(val, lhs.layout()))
             }
         }
     }
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index fbee84e09f7..b5874f62535 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -3,8 +3,6 @@ use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{Integer, Primitive};
 use rustc_target::spec::{HasTargetSpec, Target};
 
-use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef};
-
 use crate::prelude::*;
 
 pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@@ -56,11 +54,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
             FloatTy::F64 => types::F64,
         },
         ty::FnPtr(_) => pointer_ty(tcx),
-        ty::RawPtr(TypeAndMut {
-            ty: pointee_ty,
-            mutbl: _,
-        })
-        | ty::Ref(_, pointee_ty, _) => {
+        ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
             if has_ptr_meta(tcx, pointee_ty) {
                 return None;
             } else {
@@ -99,11 +93,7 @@ fn clif_pair_type_from_ty<'tcx>(
             }
             (a, b)
         }
-        ty::RawPtr(TypeAndMut {
-            ty: pointee_ty,
-            mutbl: _,
-        })
-        | ty::Ref(_, pointee_ty, _) => {
+        ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
             if has_ptr_meta(tcx, pointee_ty) {
                 (pointer_ty(tcx), pointer_ty(tcx))
             } else {
@@ -116,15 +106,8 @@ fn clif_pair_type_from_ty<'tcx>(
 
 /// Is a pointer to this type a fat ptr?
 pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
-    let ptr_ty = tcx.mk_ptr(TypeAndMut {
-        ty,
-        mutbl: rustc_hir::Mutability::Not,
-    });
-    match &tcx
-        .layout_of(ParamEnv::reveal_all().and(ptr_ty))
-        .unwrap()
-        .abi
-    {
+    let ptr_ty = tcx.mk_ptr(TypeAndMut { ty, mutbl: rustc_hir::Mutability::Not });
+    match &tcx.layout_of(ParamEnv::reveal_all().and(ptr_ty)).unwrap().abi {
         Abi::Scalar(_) => false,
         Abi::ScalarPair(_, _) => true,
         abi => unreachable!("Abi of ptr to {:?} is {:?}???", ty, abi),
@@ -132,7 +115,7 @@ pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
 }
 
 pub(crate) fn codegen_icmp_imm(
-    fx: &mut FunctionCx<'_, '_, impl Module>,
+    fx: &mut FunctionCx<'_, '_, '_>,
     intcc: IntCC,
     lhs: Value,
     rhs: i128,
@@ -175,51 +158,6 @@ pub(crate) fn codegen_icmp_imm(
     }
 }
 
-fn resolve_normal_value_imm(func: &Function, val: Value) -> Option<i64> {
-    if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) {
-        if let InstructionData::UnaryImm {
-            opcode: Opcode::Iconst,
-            imm,
-        } = func.dfg[inst]
-        {
-            Some(imm.into())
-        } else {
-            None
-        }
-    } else {
-        None
-    }
-}
-
-fn resolve_128bit_value_imm(func: &Function, val: Value) -> Option<u128> {
-    let (lsb, msb) = if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) {
-        if let InstructionData::Binary {
-            opcode: Opcode::Iconcat,
-            args: [lsb, msb],
-        } = func.dfg[inst]
-        {
-            (lsb, msb)
-        } else {
-            return None;
-        }
-    } else {
-        return None;
-    };
-
-    let lsb = u128::from(resolve_normal_value_imm(func, lsb)? as u64);
-    let msb = u128::from(resolve_normal_value_imm(func, msb)? as u64);
-
-    Some(msb << 64 | lsb)
-}
-
-pub(crate) fn resolve_value_imm(func: &Function, val: Value) -> Option<u128> {
-    if func.dfg.value_type(val) == types::I128 {
-        resolve_128bit_value_imm(func, val)
-    } else {
-        resolve_normal_value_imm(func, val).map(|imm| u128::from(imm as u64))
-    }
-}
-
 pub(crate) fn type_min_max_value(
     bcx: &mut FunctionBuilder<'_>,
     ty: Type,
@@ -288,8 +226,8 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
     }
 }
 
-pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> {
-    pub(crate) cx: &'clif mut crate::CodegenCx<'tcx, M>,
+pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
+    pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>,
     pub(crate) tcx: TyCtxt<'tcx>,
     pub(crate) pointer_type: Type, // Cached from module
 
@@ -316,7 +254,7 @@ pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> {
     pub(crate) inline_asm_index: u32,
 }
 
-impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> LayoutOf for FunctionCx<'_, '_, 'tcx> {
     type Ty = Ty<'tcx>;
     type TyAndLayout = TyAndLayout<'tcx>;
 
@@ -325,31 +263,31 @@ impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> {
     }
 }
 
-impl<'tcx, M: Module> layout::HasTyCtxt<'tcx> for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> layout::HasTyCtxt<'tcx> for FunctionCx<'_, '_, 'tcx> {
     fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
         self.tcx
     }
 }
 
-impl<'tcx, M: Module> rustc_target::abi::HasDataLayout for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> rustc_target::abi::HasDataLayout for FunctionCx<'_, '_, 'tcx> {
     fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout {
         &self.tcx.data_layout
     }
 }
 
-impl<'tcx, M: Module> layout::HasParamEnv<'tcx> for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> layout::HasParamEnv<'tcx> for FunctionCx<'_, '_, 'tcx> {
     fn param_env(&self) -> ParamEnv<'tcx> {
         ParamEnv::reveal_all()
     }
 }
 
-impl<'tcx, M: Module> HasTargetSpec for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> {
     fn target_spec(&self) -> &Target {
         &self.tcx.sess.target
     }
 }
 
-impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
+impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     pub(crate) fn monomorphize<T>(&self, value: T) -> T
     where
         T: TypeFoldable<'tcx> + Copy,
@@ -416,20 +354,14 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
         let msg_id = self
             .cx
             .module
-            .declare_data(
-                &format!("__{}_{:08x}", prefix, msg_hash),
-                Linkage::Local,
-                false,
-                false,
-            )
+            .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false)
             .unwrap();
 
         // Ignore DuplicateDefinition error, as the data will be the same
         let _ = self.cx.module.define_data(msg_id, &data_ctx);
 
         let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
             self.add_comment(local_msg_id, msg);
         }
         self.bcx.ins().global_value(self.pointer_type, local_msg_id)
@@ -444,15 +376,13 @@ impl<'tcx> LayoutOf for RevealAllLayoutCx<'tcx> {
 
     fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> {
         assert!(!ty.still_further_specializable());
-        self.0
-            .layout_of(ParamEnv::reveal_all().and(&ty))
-            .unwrap_or_else(|e| {
-                if let layout::LayoutError::SizeOverflow(_) = e {
-                    self.0.sess.fatal(&e.to_string())
-                } else {
-                    bug!("failed to get layout for `{}`: {}", ty, e)
-                }
-            })
+        self.0.layout_of(ParamEnv::reveal_all().and(&ty)).unwrap_or_else(|e| {
+            if let layout::LayoutError::SizeOverflow(_) = e {
+                self.0.sess.fatal(&e.to_string())
+            } else {
+                bug!("failed to get layout for `{}`: {}", ty, e)
+            }
+        })
     }
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
new file mode 100644
index 00000000000..177f850afb3
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
@@ -0,0 +1,41 @@
+macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) {
+    #[cfg(feature = "jit")]
+    #[allow(improper_ctypes)]
+    extern "C" {
+        $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
+    }
+
+    #[cfg(feature = "jit")]
+    pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
+        for &(name, val) in &[$((stringify!($name), $name as *const u8)),*] {
+            builder.symbol(name, val);
+        }
+    }
+}
+
+builtin_functions! {
+    register_functions_for_jit;
+
+    // integers
+    fn __multi3(a: i128, b: i128) -> i128;
+    fn __udivti3(n: u128, d: u128) -> u128;
+    fn __divti3(n: i128, d: i128) -> i128;
+    fn __umodti3(n: u128, d: u128) -> u128;
+    fn __modti3(n: i128, d: i128) -> i128;
+    fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool);
+
+    // floats
+    fn __floattisf(i: i128) -> f32;
+    fn __floattidf(i: i128) -> f64;
+    fn __floatuntisf(i: u128) -> f32;
+    fn __floatuntidf(i: u128) -> f64;
+    fn __fixsfti(f: f32) -> i128;
+    fn __fixdfti(f: f64) -> i128;
+    fn __fixunssfti(f: f32) -> u128;
+    fn __fixunsdfti(f: f64) -> u128;
+}
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index 5702832bcb6..fcd41c84465 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -8,7 +8,7 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{
     read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Pointer, Scalar,
 };
-use rustc_middle::ty::{Const, ConstKind};
+use rustc_middle::ty::ConstKind;
 
 use cranelift_codegen::ir::GlobalValueData;
 use cranelift_module::*;
@@ -28,7 +28,7 @@ enum TodoItem {
 }
 
 impl ConstantCx {
-    pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut impl Module) {
+    pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) {
         //println!("todo {:?}", self.todo);
         define_all_allocs(tcx, module, &mut self);
         //println!("done {:?}", self.done);
@@ -36,21 +36,23 @@ impl ConstantCx {
     }
 }
 
-pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, impl Module>) {
+pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
+    let mut all_constants_ok = true;
     for constant in &fx.mir.required_consts {
-        let const_ = fx.monomorphize(constant.literal);
+        let const_ = match fx.monomorphize(constant.literal) {
+            ConstantKind::Ty(ct) => ct,
+            ConstantKind::Val(..) => continue,
+        };
         match const_.val {
             ConstKind::Value(_) => {}
-            ConstKind::Unevaluated(def, ref substs, promoted) => {
+            ConstKind::Unevaluated(unevaluated) => {
                 if let Err(err) =
-                    fx.tcx
-                        .const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
+                    fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None)
                 {
+                    all_constants_ok = false;
                     match err {
                         ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => {
-                            fx.tcx
-                                .sess
-                                .span_err(constant.span, "erroneous constant encountered");
+                            fx.tcx.sess.span_err(constant.span, "erroneous constant encountered");
                         }
                         ErrorHandled::TooGeneric => {
                             span_bug!(
@@ -69,6 +71,7 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, impl Module>) {
             | ConstKind::Error(_) => unreachable!("{:?}", const_),
         }
     }
+    all_constants_ok
 }
 
 pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
@@ -76,27 +79,29 @@ pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
 }
 
 pub(crate) fn codegen_tls_ref<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     def_id: DefId,
     layout: TyAndLayout<'tcx>,
 ) -> CValue<'tcx> {
-    let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
+    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    }
     let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id);
     CValue::by_val(tls_ptr, layout)
 }
 
 fn codegen_static_ref<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     def_id: DefId,
     layout: TyAndLayout<'tcx>,
 ) -> CPlace<'tcx> {
-    let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
+    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", def_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     assert!(!layout.is_unsized(), "unsized statics aren't supported");
     assert!(
@@ -110,38 +115,28 @@ fn codegen_static_ref<'tcx>(
 }
 
 pub(crate) fn codegen_constant<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     constant: &Constant<'tcx>,
 ) -> CValue<'tcx> {
-    let const_ = fx.monomorphize(constant.literal);
+    let const_ = match fx.monomorphize(constant.literal) {
+        ConstantKind::Ty(ct) => ct,
+        ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty),
+    };
     let const_val = match const_.val {
         ConstKind::Value(const_val) => const_val,
-        ConstKind::Unevaluated(def, ref substs, promoted) if fx.tcx.is_static(def.did) => {
+        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+            if fx.tcx.is_static(def.did) =>
+        {
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
-            return codegen_static_ref(
-                fx,
-                def.did,
-                fx.layout_of(fx.monomorphize(&constant.literal.ty)),
-            )
-            .to_cvalue(fx);
+            return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx);
         }
-        ConstKind::Unevaluated(def, ref substs, promoted) => {
-            match fx
-                .tcx
-                .const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
-            {
+        ConstKind::Unevaluated(unevaluated) => {
+            match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
                 Ok(const_val) => const_val,
                 Err(_) => {
-                    fx.tcx
-                        .sess
-                        .span_err(constant.span, "erroneous constant encountered");
-                    return crate::trap::trap_unreachable_ret_value(
-                        fx,
-                        fx.layout_of(const_.ty),
-                        "erroneous constant encountered",
-                    );
+                    span_bug!(constant.span, "erroneous constant not captured by required_consts");
                 }
             }
         }
@@ -156,7 +151,7 @@ pub(crate) fn codegen_constant<'tcx>(
 }
 
 pub(crate) fn codegen_const_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     const_val: ConstValue<'tcx>,
     ty: Ty<'tcx>,
 ) -> CValue<'tcx> {
@@ -172,9 +167,7 @@ pub(crate) fn codegen_const_value<'tcx>(
             if fx.clif_type(layout.ty).is_none() {
                 let (size, align) = (layout.size, layout.align.pref);
                 let mut alloc = Allocation::from_bytes(
-                    std::iter::repeat(0)
-                        .take(size.bytes_usize())
-                        .collect::<Vec<u8>>(),
+                    std::iter::repeat(0).take(size.bytes_usize()).collect::<Vec<u8>>(),
                     align,
                 );
                 let ptr = Pointer::new(AllocId(!0), Size::ZERO); // The alloc id is never used
@@ -190,40 +183,36 @@ pub(crate) fn codegen_const_value<'tcx>(
                     let base_addr = match alloc_kind {
                         Some(GlobalAlloc::Memory(alloc)) => {
                             fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
-                            let data_id = data_id_for_alloc_id(
-                                &mut fx.cx.module,
-                                ptr.alloc_id,
-                                alloc.mutability,
-                            );
+                            let data_id =
+                                data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         Some(GlobalAlloc::Function(instance)) => {
                             let func_id =
-                                crate::abi::import_function(fx.tcx, &mut fx.cx.module, instance);
+                                crate::abi::import_function(fx.tcx, fx.cx.module, instance);
                             let local_func_id =
                                 fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
                             fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)
                         }
                         Some(GlobalAlloc::Static(def_id)) => {
                             assert!(fx.tcx.is_static(def_id));
-                            let data_id =
-                                data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
+                            let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         None => bug!("missing allocation {:?}", ptr.alloc_id),
                     };
                     let val = if ptr.offset.bytes() != 0 {
-                        fx.bcx
-                            .ins()
-                            .iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
+                        fx.bcx.ins().iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
                     } else {
                         base_addr
                     };
@@ -240,32 +229,33 @@ pub(crate) fn codegen_const_value<'tcx>(
             let ptr = pointer_for_allocation(fx, data)
                 .offset_i64(fx, i64::try_from(start).unwrap())
                 .get_addr(fx);
-            let len = fx.bcx.ins().iconst(
-                fx.pointer_type,
-                i64::try_from(end.checked_sub(start).unwrap()).unwrap(),
-            );
+            let len = fx
+                .bcx
+                .ins()
+                .iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap());
             CValue::by_val_pair(ptr, len, layout)
         }
     }
 }
 
 fn pointer_for_allocation<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     alloc: &'tcx Allocation,
 ) -> crate::pointer::Pointer {
     let alloc_id = fx.tcx.create_memory_alloc(alloc);
     fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
-    let data_id = data_id_for_alloc_id(&mut fx.cx.module, alloc_id, alloc.mutability);
+    let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
 
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     crate::pointer::Pointer::new(global_ptr)
 }
 
 fn data_id_for_alloc_id(
-    module: &mut impl Module,
+    module: &mut dyn Module,
     alloc_id: AllocId,
     mutability: rustc_hir::Mutability,
 ) -> DataId {
@@ -281,7 +271,7 @@ fn data_id_for_alloc_id(
 
 fn data_id_for_static(
     tcx: TyCtxt<'_>,
-    module: &mut impl Module,
+    module: &mut dyn Module,
     def_id: DefId,
     definition: bool,
 ) -> DataId {
@@ -304,12 +294,7 @@ fn data_id_for_static(
     } else {
         !ty.is_freeze(tcx.at(DUMMY_SP), ParamEnv::reveal_all())
     };
-    let align = tcx
-        .layout_of(ParamEnv::reveal_all().and(ty))
-        .unwrap()
-        .align
-        .pref
-        .bytes();
+    let align = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().align.pref.bytes();
 
     let attrs = tcx.codegen_fn_attrs(def_id);
 
@@ -332,17 +317,11 @@ fn data_id_for_static(
         // zero.
 
         let ref_name = format!("_rust_extern_with_linkage_{}", symbol_name);
-        let ref_data_id = module
-            .declare_data(&ref_name, Linkage::Local, false, false)
-            .unwrap();
+        let ref_data_id = module.declare_data(&ref_name, Linkage::Local, false, false).unwrap();
         let mut data_ctx = DataContext::new();
         data_ctx.set_align(align);
         let data = module.declare_data_in_data(data_id, &mut data_ctx);
-        data_ctx.define(
-            std::iter::repeat(0)
-                .take(pointer_ty(tcx).bytes() as usize)
-                .collect(),
-        );
+        data_ctx.define(std::iter::repeat(0).take(pointer_ty(tcx).bytes() as usize).collect());
         data_ctx.write_data_addr(0, data, 0);
         match module.define_data(ref_data_id, &data_ctx) {
             // Every time the static is referenced there will be another definition of this global,
@@ -356,7 +335,7 @@ fn data_id_for_static(
     }
 }
 
-fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut ConstantCx) {
+fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut ConstantCx) {
     while let Some(todo_item) = cx.todo.pop() {
         let (data_id, alloc, section_name) = match todo_item {
             TodoItem::Alloc(alloc_id) => {
@@ -371,10 +350,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
             TodoItem::Static(def_id) => {
                 //println!("static {:?}", def_id);
 
-                let section_name = tcx
-                    .codegen_fn_attrs(def_id)
-                    .link_section
-                    .map(|s| s.as_str());
+                let section_name = tcx.codegen_fn_attrs(def_id).link_section.map(|s| s.as_str());
 
                 let alloc = tcx.eval_static_initializer(def_id).unwrap();
 
@@ -396,9 +372,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
             data_ctx.set_segment_section("", &*section_name);
         }
 
-        let bytes = alloc
-            .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len())
-            .to_vec();
+        let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
         data_ctx.define(bytes.into_boxed_slice());
 
         for &(offset, (_tag, reloc)) in alloc.relocations().iter() {
@@ -426,10 +400,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
                     data_id_for_alloc_id(module, reloc, target_alloc.mutability)
                 }
                 GlobalAlloc::Static(def_id) => {
-                    if tcx
-                        .codegen_fn_attrs(def_id)
-                        .flags
-                        .contains(CodegenFnAttrFlags::THREAD_LOCAL)
+                    if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)
                     {
                         tcx.sess.fatal(&format!(
                             "Allocation {:?} contains reference to TLS value {:?}",
@@ -457,14 +428,16 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
 }
 
 pub(crate) fn mir_operand_get_const_val<'tcx>(
-    fx: &FunctionCx<'_, 'tcx, impl Module>,
+    fx: &FunctionCx<'_, '_, 'tcx>,
     operand: &Operand<'tcx>,
-) -> Option<&'tcx Const<'tcx>> {
+) -> Option<ConstValue<'tcx>> {
     match operand {
         Operand::Copy(_) | Operand::Move(_) => None,
-        Operand::Constant(const_) => Some(
-            fx.monomorphize(const_.literal)
-                .eval(fx.tcx, ParamEnv::reveal_all()),
-        ),
+        Operand::Constant(const_) => match const_.literal {
+            ConstantKind::Ty(const_) => {
+                fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value()
+            }
+            ConstantKind::Val(val, _) => Some(val),
+        },
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
index 6160f9b78d8..6018eefcd42 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
@@ -14,10 +14,7 @@ impl DebugContext<'_> {
         let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone());
         let root = self.dwarf.unit.root();
         let root = self.dwarf.unit.get_mut(root);
-        root.set(
-            gimli::DW_AT_ranges,
-            AttributeValue::RangeListRef(unit_range_list_id),
-        );
+        root.set(gimli::DW_AT_ranges, AttributeValue::RangeListRef(unit_range_list_id));
 
         let mut sections = Sections::new(WriterRelocate::new(self.endian));
         self.dwarf.write(&mut sections).unwrap();
@@ -66,10 +63,7 @@ pub(super) struct WriterRelocate {
 
 impl WriterRelocate {
     pub(super) fn new(endian: RunTimeEndian) -> Self {
-        WriterRelocate {
-            relocs: Vec::new(),
-            writer: EndianVec::new(endian),
-        }
+        WriterRelocate { relocs: Vec::new(), writer: EndianVec::new(endian) }
     }
 
     /// Perform the collected relocations to be usable for JIT usage.
@@ -85,9 +79,7 @@ impl WriterRelocate {
                         cranelift_module::FuncId::from_u32(sym.try_into().unwrap()),
                     );
                     let val = (addr as u64 as i64 + reloc.addend) as u64;
-                    self.writer
-                        .write_udata_at(reloc.offset as usize, val, reloc.size)
-                        .unwrap();
+                    self.writer.write_udata_at(reloc.offset as usize, val, reloc.size).unwrap();
                 }
             }
         }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
index d226755d85d..8578ab33ced 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
@@ -39,11 +39,11 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
     #[cfg(unix)]
     {
         use std::os::unix::ffi::OsStrExt;
-        return path.as_bytes();
+        path.as_bytes()
     }
     #[cfg(not(unix))]
     {
-        return path.to_str().unwrap().as_bytes();
+        path.to_str().unwrap().as_bytes()
     }
 }
 
@@ -53,11 +53,7 @@ pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
     if hash.kind == SourceFileHashAlgorithm::Md5 {
         let mut buf = [0u8; MD5_LEN];
         buf.copy_from_slice(hash.hash_bytes());
-        Some(FileInfo {
-            timestamp: 0,
-            size: 0,
-            md5: buf,
-        })
+        Some(FileInfo { timestamp: 0, size: 0, md5: buf })
     } else {
         None
     }
@@ -112,24 +108,14 @@ impl<'tcx> DebugContext<'tcx> {
 
         let entry = self.dwarf.unit.get_mut(entry_id);
 
-        entry.set(
-            gimli::DW_AT_decl_file,
-            AttributeValue::FileIndex(Some(file_id)),
-        );
-        entry.set(
-            gimli::DW_AT_decl_line,
-            AttributeValue::Udata(loc.line as u64),
-        );
+        entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
+        entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64));
         // FIXME: probably omit this
-        entry.set(
-            gimli::DW_AT_decl_column,
-            AttributeValue::Udata(loc.col.to_usize() as u64),
-        );
+        entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64));
     }
 
     pub(super) fn create_debug_lines(
         &mut self,
-        isa: &dyn cranelift_codegen::isa::TargetIsa,
         symbol: usize,
         entry_id: UnitEntryId,
         context: &Context,
@@ -138,7 +124,6 @@ impl<'tcx> DebugContext<'tcx> {
     ) -> CodeOffset {
         let tcx = self.tcx;
         let line_program = &mut self.dwarf.unit.line_program;
-        let func = &context.func;
 
         let line_strings = &mut self.dwarf.line_strings;
         let mut last_span = None;
@@ -202,43 +187,22 @@ impl<'tcx> DebugContext<'tcx> {
 
         let mut func_end = 0;
 
-        if let Some(ref mcr) = &context.mach_compile_result {
-            for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
-                line_program.row().address_offset = u64::from(start);
-                if !loc.is_default() {
-                    let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
-                    create_row_for_span(line_program, source_info.span);
-                } else {
-                    create_row_for_span(line_program, function_span);
-                }
-                func_end = end;
-            }
-
-            line_program.end_sequence(u64::from(func_end));
-
-            func_end = mcr.buffer.total_size();
-        } else {
-            let encinfo = isa.encoding_info();
-            let mut blocks = func.layout.blocks().collect::<Vec<_>>();
-            blocks.sort_by_key(|block| func.offsets[*block]); // Ensure inst offsets always increase
-
-            for block in blocks {
-                for (offset, inst, size) in func.inst_offsets(block, &encinfo) {
-                    let srcloc = func.srclocs[inst];
-                    line_program.row().address_offset = u64::from(offset);
-                    if !srcloc.is_default() {
-                        let source_info =
-                            *source_info_set.get_index(srcloc.bits() as usize).unwrap();
-                        create_row_for_span(line_program, source_info.span);
-                    } else {
-                        create_row_for_span(line_program, function_span);
-                    }
-                    func_end = offset + size;
-                }
+        let mcr = context.mach_compile_result.as_ref().unwrap();
+        for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
+            line_program.row().address_offset = u64::from(start);
+            if !loc.is_default() {
+                let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
+                create_row_for_span(line_program, source_info.span);
+            } else {
+                create_row_for_span(line_program, function_span);
             }
-            line_program.end_sequence(u64::from(func_end));
+            func_end = end;
         }
 
+        line_program.end_sequence(u64::from(func_end));
+
+        let func_end = mcr.buffer.total_size();
+
         assert_ne!(func_end, 0);
 
         let entry = self.dwarf.unit.get_mut(entry_id);
@@ -246,10 +210,7 @@ impl<'tcx> DebugContext<'tcx> {
             gimli::DW_AT_low_pc,
             AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
         );
-        entry.set(
-            gimli::DW_AT_high_pc,
-            AttributeValue::Udata(u64::from(func_end)),
-        );
+        entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end)));
 
         self.emit_location(entry_id, function_span);
 
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index a6f4ded41b6..dc8bc8d9cb7 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -9,7 +9,7 @@ use crate::prelude::*;
 use rustc_index::vec::IndexVec;
 
 use cranelift_codegen::entity::EntityRef;
-use cranelift_codegen::ir::{StackSlots, ValueLabel, ValueLoc};
+use cranelift_codegen::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLoc};
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::ValueLocRange;
 
@@ -39,7 +39,6 @@ pub(crate) struct DebugContext<'tcx> {
     dwarf: DwarfUnit,
     unit_range_list: RangeList,
 
-    clif_types: FxHashMap<Type, UnitEntryId>,
     types: FxHashMap<Ty<'tcx>, UnitEntryId>,
 }
 
@@ -91,20 +90,11 @@ impl<'tcx> DebugContext<'tcx> {
 
             let root = dwarf.unit.root();
             let root = dwarf.unit.get_mut(root);
-            root.set(
-                gimli::DW_AT_producer,
-                AttributeValue::StringRef(dwarf.strings.add(producer)),
-            );
-            root.set(
-                gimli::DW_AT_language,
-                AttributeValue::Language(gimli::DW_LANG_Rust),
-            );
+            root.set(gimli::DW_AT_producer, AttributeValue::StringRef(dwarf.strings.add(producer)));
+            root.set(gimli::DW_AT_language, AttributeValue::Language(gimli::DW_LANG_Rust));
             root.set(gimli::DW_AT_name, AttributeValue::StringRef(name));
             root.set(gimli::DW_AT_comp_dir, AttributeValue::StringRef(comp_dir));
-            root.set(
-                gimli::DW_AT_low_pc,
-                AttributeValue::Address(Address::Constant(0)),
-            );
+            root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0)));
         }
 
         DebugContext {
@@ -115,48 +105,10 @@ impl<'tcx> DebugContext<'tcx> {
             dwarf,
             unit_range_list: RangeList(Vec::new()),
 
-            clif_types: FxHashMap::default(),
             types: FxHashMap::default(),
         }
     }
 
-    fn dwarf_ty_for_clif_ty(&mut self, ty: Type) -> UnitEntryId {
-        if let Some(type_id) = self.clif_types.get(&ty) {
-            return *type_id;
-        }
-
-        let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag);
-
-        let primitive = |dwarf: &mut DwarfUnit, ate| {
-            let type_id = new_entry(dwarf, gimli::DW_TAG_base_type);
-            let type_entry = dwarf.unit.get_mut(type_id);
-            type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate));
-            type_id
-        };
-
-        let type_id = if ty.is_bool() {
-            primitive(&mut self.dwarf, gimli::DW_ATE_boolean)
-        } else if ty.is_int() {
-            primitive(&mut self.dwarf, gimli::DW_ATE_address)
-        } else if ty.is_float() {
-            primitive(&mut self.dwarf, gimli::DW_ATE_float)
-        } else {
-            new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type)
-        };
-
-        let type_entry = self.dwarf.unit.get_mut(type_id);
-        type_entry.set(
-            gimli::DW_AT_name,
-            AttributeValue::String(format!("{}", ty).replace('i', "u").into_bytes()),
-        );
-        type_entry.set(
-            gimli::DW_AT_byte_size,
-            AttributeValue::Udata(u64::from(ty.bytes())),
-        );
-
-        type_id
-    }
-
     fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId {
         if let Some(type_id) = self.types.get(ty) {
             return *type_id;
@@ -181,10 +133,7 @@ impl<'tcx> DebugContext<'tcx> {
             ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed),
             ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float),
             ty::Ref(_, pointee_ty, _mutbl)
-            | ty::RawPtr(ty::TypeAndMut {
-                ty: pointee_ty,
-                mutbl: _mutbl,
-            }) => {
+            | ty::RawPtr(ty::TypeAndMut { ty: pointee_ty, mutbl: _mutbl }) => {
                 let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type);
 
                 // Ensure that type is inserted before recursing to avoid duplicates
@@ -211,10 +160,7 @@ impl<'tcx> DebugContext<'tcx> {
                     let field_offset = layout.fields.offset(field_idx);
                     let field_layout = layout
                         .field(
-                            &layout::LayoutCx {
-                                tcx: self.tcx,
-                                param_env: ParamEnv::reveal_all(),
-                            },
+                            &layout::LayoutCx { tcx: self.tcx, param_env: ParamEnv::reveal_all() },
                             field_idx,
                         )
                         .unwrap();
@@ -243,10 +189,7 @@ impl<'tcx> DebugContext<'tcx> {
         let type_entry = self.dwarf.unit.get_mut(type_id);
 
         type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
-        type_entry.set(
-            gimli::DW_AT_byte_size,
-            AttributeValue::Udata(layout.size.bytes()),
-        );
+        type_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes()));
 
         self.types.insert(ty, type_id);
 
@@ -286,23 +229,15 @@ impl<'tcx> DebugContext<'tcx> {
         let name_id = self.dwarf.strings.add(name);
         // Gdb requires DW_AT_name. Otherwise the DW_TAG_subprogram is skipped.
         entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id));
-        entry.set(
-            gimli::DW_AT_linkage_name,
-            AttributeValue::StringRef(name_id),
-        );
+        entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id));
 
-        let end =
-            self.create_debug_lines(isa, symbol, entry_id, context, mir.span, source_info_set);
+        let end = self.create_debug_lines(symbol, entry_id, context, mir.span, source_info_set);
 
         self.unit_range_list.0.push(Range::StartLength {
             begin: Address::Symbol { symbol, addend: 0 },
             length: u64::from(end),
         });
 
-        if isa.get_mach_backend().is_some() {
-            return; // Not yet implemented for the AArch64 backend.
-        }
-
         let func_entry = self.dwarf.unit.get_mut(entry_id);
         // Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped.
         func_entry.set(
@@ -312,51 +247,6 @@ impl<'tcx> DebugContext<'tcx> {
         // Using Udata for DW_AT_high_pc requires at least DWARF4
         func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end)));
 
-        // FIXME Remove once actual debuginfo for locals works.
-        for (i, (param, &val)) in context
-            .func
-            .signature
-            .params
-            .iter()
-            .zip(
-                context
-                    .func
-                    .dfg
-                    .block_params(context.func.layout.entry_block().unwrap()),
-            )
-            .enumerate()
-        {
-            use cranelift_codegen::ir::ArgumentPurpose;
-            let base_name = match param.purpose {
-                ArgumentPurpose::Normal => "arg",
-                ArgumentPurpose::StructArgument(_) => "struct_arg",
-                ArgumentPurpose::StructReturn => "sret",
-                ArgumentPurpose::Link
-                | ArgumentPurpose::FramePointer
-                | ArgumentPurpose::CalleeSaved => continue,
-                ArgumentPurpose::VMContext
-                | ArgumentPurpose::SignatureId
-                | ArgumentPurpose::CallerTLS
-                | ArgumentPurpose::CalleeTLS
-                | ArgumentPurpose::StackLimit => unreachable!(),
-            };
-            let name = format!("{}{}", base_name, i);
-
-            let dw_ty = self.dwarf_ty_for_clif_ty(param.value_type);
-            let loc =
-                translate_loc(isa, context.func.locations[val], &context.func.stack_slots).unwrap();
-
-            let arg_id = self
-                .dwarf
-                .unit
-                .add(entry_id, gimli::DW_TAG_formal_parameter);
-            let var_entry = self.dwarf.unit.get_mut(arg_id);
-
-            var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
-            var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty));
-            var_entry.set(gimli::DW_AT_location, AttributeValue::Exprloc(loc));
-        }
-
         // FIXME make it more reliable and implement scopes before re-enabling this.
         if false {
             let value_labels_ranges = context.build_value_labels_ranges(isa).unwrap();
@@ -376,10 +266,7 @@ impl<'tcx> DebugContext<'tcx> {
                     context,
                     &local_map,
                     &value_labels_ranges,
-                    Place {
-                        local,
-                        projection: ty::List::empty(),
-                    },
+                    Place { local, projection: ty::List::empty() },
                 );
 
                 let var_entry = self.dwarf.unit.get_mut(var_id);
@@ -417,10 +304,7 @@ fn place_location<'tcx>(
                                 symbol,
                                 addend: i64::from(value_loc_range.start),
                             },
-                            end: Address::Symbol {
-                                symbol,
-                                addend: i64::from(value_loc_range.end),
-                            },
+                            end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
                             data: translate_loc(
                                 isa,
                                 value_loc_range.loc,
@@ -463,17 +347,17 @@ fn place_location<'tcx>(
 // Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
 fn translate_loc(
     isa: &dyn TargetIsa,
-    loc: ValueLoc,
+    loc: LabelValueLoc,
     stack_slots: &StackSlots,
 ) -> Option<Expression> {
     match loc {
-        ValueLoc::Reg(reg) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => {
             let machine_reg = isa.map_dwarf_register(reg).unwrap();
             let mut expr = Expression::new();
             expr.op_reg(gimli::Register(machine_reg));
             Some(expr)
         }
-        ValueLoc::Stack(ss) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
             if let Some(ss_offset) = stack_slots[ss].offset {
                 let mut expr = Expression::new();
                 expr.op_breg(X86_64::RBP, i64::from(ss_offset) + 16);
@@ -482,6 +366,17 @@ fn translate_loc(
                 None
             }
         }
-        _ => None,
+        LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => unreachable!(),
+        LabelValueLoc::Reg(reg) => {
+            let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
+            let mut expr = Expression::new();
+            expr.op_reg(gimli::Register(machine_reg));
+            Some(expr)
+        }
+        LabelValueLoc::SPOffset(offset) => {
+            let mut expr = Expression::new();
+            expr.op_breg(X86_64::RSP, offset);
+            Some(expr)
+        }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
index 49de927cdba..357c9fe6ed8 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
@@ -28,11 +28,7 @@ impl<'tcx> UnwindContext<'tcx> {
             None
         };
 
-        UnwindContext {
-            tcx,
-            frame_table,
-            cie_id,
-        }
+        UnwindContext { tcx, frame_table, cie_id }
     }
 
     pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
@@ -46,10 +42,8 @@ impl<'tcx> UnwindContext<'tcx> {
             UnwindInfo::SystemV(unwind_info) => {
                 self.frame_table.add_fde(
                     self.cie_id.unwrap(),
-                    unwind_info.to_fde(Address::Symbol {
-                        symbol: func_id.as_u32() as usize,
-                        addend: 0,
-                    }),
+                    unwind_info
+                        .to_fde(Address::Symbol { symbol: func_id.as_u32() as usize, addend: 0 }),
                 );
             }
             UnwindInfo::WindowsX64(_) => {
@@ -60,9 +54,8 @@ impl<'tcx> UnwindContext<'tcx> {
     }
 
     pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) {
-        let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
-            self.tcx,
-        )));
+        let mut eh_frame =
+            EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
         self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
 
         if !eh_frame.0.writer.slice().is_empty() {
@@ -82,9 +75,8 @@ impl<'tcx> UnwindContext<'tcx> {
         self,
         jit_module: &cranelift_jit::JITModule,
     ) -> Option<UnwindRegistry> {
-        let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
-            self.tcx,
-        )));
+        let mut eh_frame =
+            EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
         self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
 
         if eh_frame.0.writer.slice().is_empty() {
@@ -130,10 +122,7 @@ impl<'tcx> UnwindContext<'tcx> {
             registrations.push(ptr as usize);
         }
 
-        Some(UnwindRegistry {
-            _frame_table: eh_frame,
-            registrations,
-        })
+        Some(UnwindRegistry { _frame_table: eh_frame, registrations })
     }
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs
index ad635016a91..3326f87f000 100644
--- a/compiler/rustc_codegen_cranelift/src/discriminant.rs
+++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs
@@ -7,7 +7,7 @@ use rustc_target::abi::{Int, TagEncoding, Variants};
 use crate::prelude::*;
 
 pub(crate) fn codegen_set_discriminant<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     place: CPlace<'tcx>,
     variant_index: VariantIdx,
 ) {
@@ -26,11 +26,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
             variants: _,
         } => {
             let ptr = place.place_field(fx, mir::Field::new(tag_field));
-            let to = layout
-                .ty
-                .discriminant_for_variant(fx.tcx, variant_index)
-                .unwrap()
-                .val;
+            let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val;
             let to = if ptr.layout().abi.is_signed() {
                 ty::ScalarInt::try_from_int(
                     ptr.layout().size.sign_extend(to) as i128,
@@ -46,12 +42,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
         Variants::Multiple {
             tag: _,
             tag_field,
-            tag_encoding:
-                TagEncoding::Niche {
-                    dataful_variant,
-                    ref niche_variants,
-                    niche_start,
-                },
+            tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
             variants: _,
         } => {
             if variant_index != dataful_variant {
@@ -70,7 +61,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
 }
 
 pub(crate) fn codegen_get_discriminant<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     value: CValue<'tcx>,
     dest_layout: TyAndLayout<'tcx>,
 ) -> CValue<'tcx> {
@@ -101,12 +92,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             };
             return CValue::const_val(fx, dest_layout, discr_val);
         }
-        Variants::Multiple {
-            tag,
-            tag_field,
-            tag_encoding,
-            variants: _,
-        } => (tag, *tag_field, tag_encoding),
+        Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => {
+            (tag, *tag_field, tag_encoding)
+        }
     };
 
     let cast_to = fx.clif_type(dest_layout.ty).unwrap();
@@ -125,11 +113,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             let val = clif_intcast(fx, tag, cast_to, signed);
             CValue::by_val(val, dest_layout)
         }
-        TagEncoding::Niche {
-            dataful_variant,
-            ref niche_variants,
-            niche_start,
-        } => {
+        TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
             // Rebase from niche values to discriminants, and check
             // whether the result is in range for the niche variants.
 
@@ -146,9 +130,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
                 tag
             } else {
                 // FIXME handle niche_start > i64::MAX
-                fx.bcx
-                    .ins()
-                    .iadd_imm(tag, -i64::try_from(niche_start).unwrap())
+                fx.bcx.ins().iadd_imm(tag, -i64::try_from(niche_start).unwrap())
             };
             let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
             let is_niche = {
@@ -176,15 +158,10 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
                 } else {
                     clif_intcast(fx, relative_discr, cast_to, false)
                 };
-                fx.bcx
-                    .ins()
-                    .iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
+                fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
             };
 
-            let dataful_variant = fx
-                .bcx
-                .ins()
-                .iconst(cast_to, i64::from(dataful_variant.as_u32()));
+            let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
             let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
             CValue::by_val(discr, dest_layout)
         }
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index 39781e2482a..ed3bdedddce 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -12,11 +12,9 @@ use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{DebugInfo, OutputType};
 
-use cranelift_object::{ObjectModule, ObjectProduct};
+use cranelift_object::ObjectModule;
 
-use crate::prelude::*;
-
-use crate::backend::AddConstructor;
+use crate::{prelude::*, BackendConfig};
 
 fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule {
     let module = crate::backend::make_module(tcx.sess, name);
@@ -39,7 +37,6 @@ fn emit_module(
     module: ObjectModule,
     debug: Option<DebugContext<'_>>,
     unwind_context: UnwindContext<'_>,
-    map_product: impl FnOnce(ObjectProduct) -> ObjectProduct,
 ) -> ModuleCodegenResult {
     let mut product = module.finish();
 
@@ -49,15 +46,10 @@ fn emit_module(
 
     unwind_context.emit(&mut product);
 
-    let product = map_product(product);
-
-    let tmp_file = tcx
-        .output_filenames(LOCAL_CRATE)
-        .temp_path(OutputType::Object, Some(&name));
+    let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name));
     let obj = product.object.write().unwrap();
     if let Err(err) = std::fs::write(&tmp_file, obj) {
-        tcx.sess
-            .fatal(&format!("error writing object file: {}", err));
+        tcx.sess.fatal(&format!("error writing object file: {}", err));
     }
 
     let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() {
@@ -71,13 +63,7 @@ fn emit_module(
     };
 
     ModuleCodegenResult(
-        CompiledModule {
-            name,
-            kind,
-            object: Some(tmp_file),
-            dwarf_object: None,
-            bytecode: None,
-        },
+        CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None },
         work_product,
     )
 }
@@ -117,49 +103,26 @@ fn reuse_workproduct_for_cgu(
     }
 }
 
-fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodegenResult {
+fn module_codegen(
+    tcx: TyCtxt<'_>,
+    (backend_config, cgu_name): (BackendConfig, rustc_span::Symbol),
+) -> ModuleCodegenResult {
     let cgu = tcx.codegen_unit(cgu_name);
     let mono_items = cgu.items_in_deterministic_order(tcx);
 
     let mut module = new_module(tcx, cgu_name.as_str().to_string());
 
-    // Initialize the global atomic mutex using a constructor for proc-macros.
-    // FIXME implement atomic instructions in Cranelift.
-    let mut init_atomics_mutex_from_constructor = None;
-    if tcx
-        .sess
-        .crate_types()
-        .contains(&rustc_session::config::CrateType::ProcMacro)
-    {
-        if mono_items.iter().any(|(mono_item, _)| match mono_item {
-            rustc_middle::mir::mono::MonoItem::Static(def_id) => tcx
-                .symbol_name(Instance::mono(tcx, *def_id))
-                .name
-                .contains("__rustc_proc_macro_decls_"),
-            _ => false,
-        }) {
-            init_atomics_mutex_from_constructor =
-                Some(crate::atomic_shim::init_global_lock_constructor(
-                    &mut module,
-                    &format!("{}_init_atomics_mutex", cgu_name.as_str()),
-                ));
-        }
-    }
-
     let mut cx = crate::CodegenCx::new(
         tcx,
-        module,
+        backend_config,
+        &mut module,
         tcx.sess.opts.debuginfo != DebugInfo::None,
-        true,
     );
     super::predefine_mono_items(&mut cx, &mono_items);
-    for (mono_item, (linkage, visibility)) in mono_items {
-        let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+    for (mono_item, _) in mono_items {
         match mono_item {
             MonoItem::Fn(inst) => {
-                cx.tcx.sess.time("codegen fn", || {
-                    crate::base::codegen_fn(&mut cx, inst, linkage)
-                });
+                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
             }
             MonoItem::Static(def_id) => {
                 crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@@ -175,9 +138,9 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
             }
         }
     }
-    let (mut module, global_asm, debug, mut unwind_context) =
+    let (global_asm, debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context, false);
+    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context);
 
     let codegen_result = emit_module(
         tcx,
@@ -186,13 +149,6 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
         module,
         debug,
         unwind_context,
-        |mut product| {
-            if let Some(func_id) = init_atomics_mutex_from_constructor {
-                product.add_constructor(func_id);
-            }
-
-            product
-        },
     );
 
     codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm);
@@ -202,9 +158,25 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
 
 pub(super) fn run_aot(
     tcx: TyCtxt<'_>,
+    backend_config: BackendConfig,
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
+    use rustc_span::symbol::sym;
+
+    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
+    let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
+    let windows_subsystem = subsystem.map(|subsystem| {
+        if subsystem != sym::windows && subsystem != sym::console {
+            tcx.sess.fatal(&format!(
+                "invalid windows subsystem `{}`, only \
+                                    `windows` and `console` are allowed",
+                subsystem
+            ));
+        }
+        subsystem.to_string()
+    });
+
     let mut work_products = FxHashMap::default();
 
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
@@ -225,9 +197,7 @@ pub(super) fn run_aot(
         cgus.iter()
             .map(|cgu| {
                 let cgu_reuse = determine_cgu_reuse(tcx, cgu);
-                tcx.sess
-                    .cgu_reuse_tracker
-                    .set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
+                tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
 
                 match cgu_reuse {
                     _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {}
@@ -242,7 +212,7 @@ pub(super) fn run_aot(
                 let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
                     dep_node,
                     tcx,
-                    cgu.name(),
+                    (backend_config, cgu.name()),
                     module_codegen,
                     rustc_middle::dep_graph::hash_result,
                 );
@@ -271,7 +241,6 @@ pub(super) fn run_aot(
             allocator_module,
             None,
             allocator_unwind_context,
-            |product| product,
         );
         if let Some((id, product)) = work_product {
             work_products.insert(id, product);
@@ -301,8 +270,7 @@ pub(super) fn run_aot(
             });
 
             if let Err(err) = std::fs::write(&tmp_file, obj) {
-                tcx.sess
-                    .fatal(&format!("error writing metadata object file: {}", err));
+                tcx.sess.fatal(&format!("error writing metadata object file: {}", err));
             }
 
             (metadata_cgu_name, tmp_file)
@@ -326,7 +294,7 @@ pub(super) fn run_aot(
             allocator_module,
             metadata_module,
             metadata,
-            windows_subsystem: None, // Windows is not yet supported
+            windows_subsystem,
             linker_info: LinkerInfo::new(tcx),
             crate_info: CrateInfo::new(tcx),
         },
@@ -356,8 +324,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
                 "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift",
             );
         } else {
-            tcx.sess
-                .fatal("asm! and global_asm! are not yet supported on macOS and Windows");
+            tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows");
         }
     }
 
@@ -367,19 +334,12 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
     // Remove all LLVM style comments
     let global_asm = global_asm
         .lines()
-        .map(|line| {
-            if let Some(index) = line.find("//") {
-                &line[0..index]
-            } else {
-                line
-            }
-        })
+        .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
         .collect::<Vec<_>>()
         .join("\n");
 
-    let output_object_file = tcx
-        .output_filenames(LOCAL_CRATE)
-        .temp_path(OutputType::Object, Some(cgu_name));
+    let output_object_file =
+        tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name));
 
     // Assemble `global_asm`
     let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
@@ -389,16 +349,10 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
         .stdin(Stdio::piped())
         .spawn()
         .expect("Failed to spawn `as`.");
-    child
-        .stdin
-        .take()
-        .unwrap()
-        .write_all(global_asm.as_bytes())
-        .unwrap();
+    child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
     let status = child.wait().expect("Failed to wait for `as`.");
     if !status.success() {
-        tcx.sess
-            .fatal(&format!("Failed to assemble `{}`", global_asm));
+        tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm));
     }
 
     // Link the global asm and main object file together
@@ -442,11 +396,7 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
     }
 
     let work_product_id = &cgu.work_product_id();
-    if tcx
-        .dep_graph
-        .previous_work_product(work_product_id)
-        .is_none()
-    {
+    if tcx.dep_graph.previous_work_product(work_product_id).is_none() {
         // We don't have anything cached for this CGU. This can happen
         // if the CGU did not exist in the previous session.
         return CguReuse::No;
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index f784d8d27cc..dbe1ff083f0 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -5,66 +5,36 @@ use std::cell::RefCell;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
+use rustc_session::config::EntryFnType;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
-use crate::prelude::*;
+use crate::{prelude::*, BackendConfig};
 use crate::{CodegenCx, CodegenMode};
 
 thread_local! {
+    pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None);
     pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
 }
 
-pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
+pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     if !tcx.sess.opts.output_types.should_codegen() {
         tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
     }
 
-    #[cfg(unix)]
-    unsafe {
-        // When not using our custom driver rustc will open us without the RTLD_GLOBAL flag, so
-        // __cg_clif_global_atomic_mutex will not be exported. We fix this by opening ourself again
-        // as global.
-        // FIXME remove once atomic_shim is gone
-
-        let mut dl_info: libc::Dl_info = std::mem::zeroed();
-        assert_ne!(
-            libc::dladdr(run_jit as *const libc::c_void, &mut dl_info),
-            0
-        );
-        assert_ne!(
-            libc::dlopen(dl_info.dli_fname, libc::RTLD_NOW | libc::RTLD_GLOBAL),
-            std::ptr::null_mut(),
-        );
-    }
-
     let imported_symbols = load_imported_symbols_for_jit(tcx);
 
-    let mut jit_builder = JITBuilder::with_isa(
-        crate::build_isa(tcx.sess),
-        cranelift_module::default_libcall_names(),
-    );
-    jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy));
+    let mut jit_builder =
+        JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
+    jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
+    crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
     jit_builder.symbols(imported_symbols);
     let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
-    let sig = Signature {
-        params: vec![
-            AbiParam::new(jit_module.target_config().pointer_type()),
-            AbiParam::new(jit_module.target_config().pointer_type()),
-        ],
-        returns: vec![AbiParam::new(
-            jit_module.target_config().pointer_type(), /*isize*/
-        )],
-        call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
-    };
-    let main_func_id = jit_module
-        .declare_function("main", Linkage::Import, &sig)
-        .unwrap();
-
     let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
     let mono_items = cgus
         .iter()
@@ -74,19 +44,16 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
         .into_iter()
         .collect::<Vec<(_, (_, _))>>();
 
-    let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+    let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false);
 
     super::time(tcx, "codegen mono items", || {
         super::predefine_mono_items(&mut cx, &mono_items);
-        for (mono_item, (linkage, visibility)) in mono_items {
-            let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+        for (mono_item, _) in mono_items {
             match mono_item {
-                MonoItem::Fn(inst) => match codegen_mode {
+                MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        cx.tcx.sess.time("codegen fn", || {
-                            crate::base::codegen_fn(&mut cx, inst, linkage)
-                        });
+                        cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
                     }
                     CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
                 },
@@ -101,7 +68,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
         }
     });
 
-    let (mut jit_module, global_asm, _debug, mut unwind_context) =
+    let (global_asm, _debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
     jit_module.finalize_definitions();
 
@@ -109,21 +76,16 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
         tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
 
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
     tcx.sess.abort_if_errors();
 
     jit_module.finalize_definitions();
-
     let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
 
-    let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
-
-    println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed");
-
-    let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
-        unsafe { ::std::mem::transmute(finalized_main) };
+    println!(
+        "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
+    );
 
     let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
     let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
@@ -136,12 +98,61 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
     // useful as some dynamic linkers use it as a marker to jump over.
     argv.push(std::ptr::null());
 
-    CURRENT_MODULE
-        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
+    BACKEND_CONFIG.with(|tls_backend_config| {
+        assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
+    });
+
+    let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
+    let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx);
+
+    match entry_ty {
+        EntryFnType::Main => {
+            // FIXME set program arguments somehow
+
+            let main_sig = Signature {
+                params: vec![],
+                returns: vec![],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let main_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
+                .unwrap();
+            let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
+            f();
+            std::process::exit(0);
+        }
+        EntryFnType::Start => {
+            let start_sig = Signature {
+                params: vec![
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                ],
+                returns: vec![AbiParam::new(
+                    jit_module.target_config().pointer_type(), /*isize*/
+                )],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let start_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
+                .unwrap();
+            let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
 
-    let ret = f(args.len() as c_int, argv.as_ptr());
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
 
-    std::process::exit(ret);
+            let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
+                unsafe { ::std::mem::transmute(finalized_start) };
+            let ret = f(args.len() as c_int, argv.as_ptr());
+            std::process::exit(ret);
+        }
+    }
 }
 
 #[no_mangle]
@@ -153,21 +164,18 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
         CURRENT_MODULE.with(|jit_module| {
             let mut jit_module = jit_module.borrow_mut();
             let jit_module = jit_module.as_mut().unwrap();
-            let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+            let backend_config =
+                BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap());
 
             let name = tcx.symbol_name(instance).name.to_string();
-            let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance);
-            let func_id = cx
-                .module
-                .declare_function(&name, Linkage::Export, &sig)
-                .unwrap();
-            cx.module.prepare_for_function_redefine(func_id).unwrap();
+            let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
+            let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap();
+            jit_module.prepare_for_function_redefine(func_id).unwrap();
 
-            tcx.sess.time("codegen fn", || {
-                crate::base::codegen_fn(&mut cx, instance, Linkage::Export)
-            });
+            let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
+            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
 
-            let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize();
+            let (global_asm, _debug_context, unwind_context) = cx.finalize();
             assert!(global_asm.is_empty());
             jit_module.finalize_definitions();
             std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
@@ -194,9 +202,8 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
             Linkage::NotLinked | Linkage::IncludedFromDylib => {}
             Linkage::Static => {
                 let name = tcx.crate_name(cnum);
-                let mut err = tcx
-                    .sess
-                    .struct_err(&format!("Can't load static lib {}", name.as_str()));
+                let mut err =
+                    tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
                 err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
                 err.emit();
             }
@@ -217,6 +224,11 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
             if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
                 return None;
             }
+            if name.starts_with("rust_metadata_") {
+                // The metadata is part of a section that is not loaded by the dynamic linker in
+                // case of cg_llvm.
+                return None;
+            }
             let dlsym_name = if cfg!(target_os = "macos") {
                 // On macOS `dlsym` expects the name without leading `_`.
                 assert!(name.starts_with('_'), "{:?}", name);
@@ -236,17 +248,14 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     imported_symbols
 }
 
-pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) {
+fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let pointer_type = cx.module.target_config().pointer_type();
 
     let name = tcx.symbol_name(inst).name.to_string();
     let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
-    let func_id = cx
-        .module
-        .declare_function(&name, Linkage::Export, &sig)
-        .unwrap();
+    let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap();
 
     let instance_ptr = Box::into_raw(Box::new(inst));
 
@@ -267,28 +276,18 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In
     let mut builder_ctx = FunctionBuilderContext::new();
     let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
 
-    let jit_fn = cx
-        .module
-        .declare_func_in_func(jit_fn, trampoline_builder.func);
+    let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func);
     let sig_ref = trampoline_builder.func.import_signature(sig);
 
     let entry_block = trampoline_builder.create_block();
     trampoline_builder.append_block_params_for_function_params(entry_block);
-    let fn_args = trampoline_builder
-        .func
-        .dfg
-        .block_params(entry_block)
-        .to_vec();
+    let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec();
 
     trampoline_builder.switch_to_block(entry_block);
-    let instance_ptr = trampoline_builder
-        .ins()
-        .iconst(pointer_type, instance_ptr as u64 as i64);
+    let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64);
     let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
     let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
-    let call_inst = trampoline_builder
-        .ins()
-        .call_indirect(sig_ref, jitted_fn, &fn_args);
+    let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args);
     let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
     trampoline_builder.ins().return_(&ret_vals);
 
@@ -296,7 +295,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In
         .define_function(
             func_id,
             &mut Context::for_function(trampoline),
-            &mut cranelift_codegen::binemit::NullTrapSink {},
+            &mut NullTrapSink {},
+            &mut NullStackMapSink {},
         )
         .unwrap();
 }
diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs
index 2497f9dfdfb..d49182a07b7 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs
@@ -17,43 +17,46 @@ pub(crate) fn codegen_crate(
     tcx: TyCtxt<'_>,
     metadata: EncodedMetadata,
     need_metadata_module: bool,
-    config: crate::BackendConfig,
+    backend_config: crate::BackendConfig,
 ) -> Box<dyn Any> {
     tcx.sess.abort_if_errors();
 
-    match config.codegen_mode {
-        CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module),
+    match backend_config.codegen_mode {
+        CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module),
         CodegenMode::Jit | CodegenMode::JitLazy => {
-            let is_executable = tcx
-                .sess
-                .crate_types()
-                .contains(&rustc_session::config::CrateType::Executable);
+            let is_executable =
+                tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable);
             if !is_executable {
                 tcx.sess.fatal("can't jit non-executable crate");
             }
 
             #[cfg(feature = "jit")]
-            let _: ! = jit::run_jit(tcx, config.codegen_mode);
+            let _: ! = jit::run_jit(tcx, backend_config);
 
             #[cfg(not(feature = "jit"))]
-            tcx.sess
-                .fatal("jit support was disabled when compiling rustc_codegen_cranelift");
+            tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
         }
     }
 }
 
 fn predefine_mono_items<'tcx>(
-    cx: &mut crate::CodegenCx<'tcx, impl Module>,
+    cx: &mut crate::CodegenCx<'_, 'tcx>,
     mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
     cx.tcx.sess.time("predefine functions", || {
+        let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
         for &(mono_item, (linkage, visibility)) in mono_items {
             match mono_item {
                 MonoItem::Fn(instance) => {
                     let name = cx.tcx.symbol_name(instance).name.to_string();
                     let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
                     let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
-                    let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+                    let linkage = crate::linkage::get_clif_linkage(
+                        mono_item,
+                        linkage,
+                        visibility,
+                        is_compiler_builtins,
+                    );
                     cx.module.declare_function(&name, linkage, &sig).unwrap();
                 }
                 MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
@@ -63,21 +66,12 @@ fn predefine_mono_items<'tcx>(
 }
 
 fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
-    if std::env::var("CG_CLIF_DISPLAY_CG_TIME")
-        .as_ref()
-        .map(|val| &**val)
-        == Ok("1")
-    {
+    if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") {
         println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
         let before = std::time::Instant::now();
         let res = tcx.sess.time(name, f);
         let after = std::time::Instant::now();
-        println!(
-            "[{:<30}: {}] end time: {:?}",
-            tcx.crate_name(LOCAL_CRATE),
-            name,
-            after - before
-        );
+        println!("[{:<30}: {}] end time: {:?}", tcx.crate_name(LOCAL_CRATE), name, after - before);
         res
     } else {
         tcx.sess.time(name, f)
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 04aac780125..1fb5e86aed7 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -9,7 +9,7 @@ use rustc_middle::mir::InlineAsmOperand;
 use rustc_target::asm::*;
 
 pub(crate) fn codegen_inline_asm<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     _span: Span,
     template: &[InlineAsmTemplatePiece],
     operands: &[InlineAsmOperand<'tcx>],
@@ -20,6 +20,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     if template.is_empty() {
         // Black box
         return;
+    } else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
+        let true_ = fx.bcx.ins().iconst(types::I32, 1);
+        fx.bcx.ins().trapnz(true_, TrapCode::User(1));
+        return;
     }
 
     let mut slot_size = Size::from_bytes(0);
@@ -53,11 +57,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                     crate::base::codegen_operand(fx, value).load_scalar(fx),
                 ));
             }
-            InlineAsmOperand::Out {
-                reg,
-                late: _,
-                place,
-            } => {
+            InlineAsmOperand::Out { reg, late: _, place } => {
                 let reg = expect_reg(reg);
                 clobbered_regs.push((reg, new_slot(reg.reg_class())));
                 if let Some(place) = place {
@@ -68,12 +68,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                     ));
                 }
             }
-            InlineAsmOperand::InOut {
-                reg,
-                late: _,
-                ref in_value,
-                out_place,
-            } => {
+            InlineAsmOperand::InOut { reg, late: _, ref in_value, out_place } => {
                 let reg = expect_reg(reg);
                 clobbered_regs.push((reg, new_slot(reg.reg_class())));
                 inputs.push((
@@ -97,11 +92,8 @@ pub(crate) fn codegen_inline_asm<'tcx>(
 
     let inline_asm_index = fx.inline_asm_index;
     fx.inline_asm_index += 1;
-    let asm_name = format!(
-        "{}__inline_asm_{}",
-        fx.tcx.symbol_name(fx.instance).name,
-        inline_asm_index
-    );
+    let asm_name =
+        format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index);
 
     let generated_asm = generate_asm_wrapper(
         &asm_name,
@@ -129,12 +121,7 @@ fn generate_asm_wrapper(
     let mut generated_asm = String::new();
     writeln!(generated_asm, ".globl {}", asm_name).unwrap();
     writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
-    writeln!(
-        generated_asm,
-        ".section .text.{},\"ax\",@progbits",
-        asm_name
-    )
-    .unwrap();
+    writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
     writeln!(generated_asm, "{}:", asm_name).unwrap();
 
     generated_asm.push_str(".intel_syntax noprefix\n");
@@ -164,11 +151,7 @@ fn generate_asm_wrapper(
             InlineAsmTemplatePiece::String(s) => {
                 generated_asm.push_str(s);
             }
-            InlineAsmTemplatePiece::Placeholder {
-                operand_idx: _,
-                modifier: _,
-                span: _,
-            } => todo!(),
+            InlineAsmTemplatePiece::Placeholder { operand_idx: _, modifier: _, span: _ } => todo!(),
         }
     }
     generated_asm.push('\n');
@@ -203,7 +186,7 @@ fn generate_asm_wrapper(
 }
 
 fn call_inline_asm<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     asm_name: &str,
     slot_size: Size,
     inputs: Vec<(InlineAsmReg, Size, Value)>,
@@ -214,8 +197,9 @@ fn call_inline_asm<'tcx>(
         offset: None,
         size: u32::try_from(slot_size.bytes()).unwrap(),
     });
-    #[cfg(debug_assertions)]
-    fx.add_comment(stack_slot, "inline asm scratch slot");
+    if fx.clif_comments.enabled() {
+        fx.add_comment(stack_slot, "inline asm scratch slot");
+    }
 
     let inline_asm_func = fx
         .cx
@@ -230,17 +214,13 @@ fn call_inline_asm<'tcx>(
             },
         )
         .unwrap();
-    let inline_asm_func = fx
-        .cx
-        .module
-        .declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(inline_asm_func, asm_name);
+    let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
+    if fx.clif_comments.enabled() {
+        fx.add_comment(inline_asm_func, asm_name);
+    }
 
     for (_reg, offset, value) in inputs {
-        fx.bcx
-            .ins()
-            .stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
     }
 
     let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
@@ -248,10 +228,7 @@ fn call_inline_asm<'tcx>(
 
     for (_reg, offset, place) in outputs {
         let ty = fx.clif_type(place.layout().ty).unwrap();
-        let value = fx
-            .bcx
-            .ins()
-            .stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }
@@ -267,8 +244,7 @@ fn save_register(generated_asm: &mut String, arch: InlineAsmArch, reg: InlineAsm
     match arch {
         InlineAsmArch::X86_64 => {
             write!(generated_asm, "    mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
-            reg.emit(generated_asm, InlineAsmArch::X86_64, None)
-                .unwrap();
+            reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
             generated_asm.push('\n');
         }
         _ => unimplemented!("save_register for {:?}", arch),
@@ -284,8 +260,7 @@ fn restore_register(
     match arch {
         InlineAsmArch::X86_64 => {
             generated_asm.push_str("    mov ");
-            reg.emit(generated_asm, InlineAsmArch::X86_64, None)
-                .unwrap();
+            reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
             writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
         }
         _ => unimplemented!("restore_register for {:?}", arch),
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
index c1a1cdbe4eb..b27b0eddfba 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
@@ -6,7 +6,7 @@ use crate::prelude::*;
 ///
 /// This emulates an intel cpu with sse and sse2 support, but which doesn't support anything else.
 pub(crate) fn codegen_cpuid_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     leaf: Value,
     _subleaf: Value,
 ) -> (Value, Value, Value, Value) {
@@ -31,54 +31,28 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
 
     fx.bcx.switch_to_block(leaf_0);
     let max_basic_leaf = fx.bcx.ins().iconst(types::I32, 1);
-    let vend0 = fx
-        .bcx
-        .ins()
-        .iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu")));
-    let vend2 = fx
-        .bcx
-        .ins()
-        .iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI")));
-    let vend1 = fx
-        .bcx
-        .ins()
-        .iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel")));
-    fx.bcx
-        .ins()
-        .jump(dest, &[max_basic_leaf, vend0, vend1, vend2]);
+    let vend0 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu")));
+    let vend2 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI")));
+    let vend1 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel")));
+    fx.bcx.ins().jump(dest, &[max_basic_leaf, vend0, vend1, vend2]);
 
     fx.bcx.switch_to_block(leaf_1);
     let cpu_signature = fx.bcx.ins().iconst(types::I32, 0);
     let additional_information = fx.bcx.ins().iconst(types::I32, 0);
     let ecx_features = fx.bcx.ins().iconst(types::I32, 0);
-    let edx_features = fx
-        .bcx
-        .ins()
-        .iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
-    fx.bcx.ins().jump(
-        dest,
-        &[
-            cpu_signature,
-            additional_information,
-            ecx_features,
-            edx_features,
-        ],
-    );
+    let edx_features = fx.bcx.ins().iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
+    fx.bcx.ins().jump(dest, &[cpu_signature, additional_information, ecx_features, edx_features]);
 
     fx.bcx.switch_to_block(leaf_8000_0000);
     let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0);
     let zero = fx.bcx.ins().iconst(types::I32, 0);
-    fx.bcx
-        .ins()
-        .jump(dest, &[extended_max_basic_leaf, zero, zero, zero]);
+    fx.bcx.ins().jump(dest, &[extended_max_basic_leaf, zero, zero, zero]);
 
     fx.bcx.switch_to_block(leaf_8000_0001);
     let zero = fx.bcx.ins().iconst(types::I32, 0);
     let proc_info_ecx = fx.bcx.ins().iconst(types::I32, 0);
     let proc_info_edx = fx.bcx.ins().iconst(types::I32, 0);
-    fx.bcx
-        .ins()
-        .jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
+    fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
 
     fx.bcx.switch_to_block(unsupported_leaf);
     crate::trap::trap_unreachable(
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
index d58e4d49958..83c91f789cd 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
@@ -6,7 +6,7 @@ use crate::prelude::*;
 use rustc_middle::ty::subst::SubstsRef;
 
 pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: &str,
     substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
@@ -53,7 +53,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         };
         llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) {
             let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
-            let flt_cc = match kind_const.val.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
+            let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
                 0 => FloatCC::Equal,
                 1 => FloatCC::LessThan,
                 2 => FloatCC::LessThanOrEqual,
@@ -84,7 +84,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         llvm.x86.sse2.psrli.d, (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.val.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
                 };
@@ -94,7 +94,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         llvm.x86.sse2.pslli.d, (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.val.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
                 };
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index 8946ac43bc6..39e047a98f9 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -9,6 +9,7 @@ pub(crate) use cpuid::codegen_cpuid_call;
 pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
 use crate::prelude::*;
+use cranelift_codegen::ir::AtomicRmwOp;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 
 macro intrinsic_pat {
@@ -112,38 +113,6 @@ macro call_intrinsic_match {
     }
 }
 
-macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
-    crate::atomic_shim::lock_global_lock($fx);
-
-    let clif_ty = $fx.clif_type($T).unwrap();
-    let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
-    let new = $fx.bcx.ins().$op(old, $src);
-    $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
-    $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
-
-    crate::atomic_shim::unlock_global_lock($fx);
-}
-
-macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
-    crate::atomic_shim::lock_global_lock($fx);
-
-    // Read old
-    let clif_ty = $fx.clif_type($T).unwrap();
-    let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
-
-    // Compare
-    let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
-    let new = $fx.bcx.ins().select(is_eq, old, $src);
-
-    // Write new
-    $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
-
-    let ret_val = CValue::by_val(old, $ret.layout());
-    $ret.write_cvalue($fx, ret_val);
-
-    crate::atomic_shim::unlock_global_lock($fx);
-}
-
 macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
     match $ty.kind() {
         ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@@ -184,12 +153,12 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx
     }
 }
 
-fn simd_for_each_lane<'tcx, M: Module>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+fn simd_for_each_lane<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
     f: impl Fn(
-        &mut FunctionCx<'_, 'tcx, M>,
+        &mut FunctionCx<'_, '_, 'tcx>,
         TyAndLayout<'tcx>,
         TyAndLayout<'tcx>,
         Value,
@@ -213,13 +182,13 @@ fn simd_for_each_lane<'tcx, M: Module>(
     }
 }
 
-fn simd_pair_for_each_lane<'tcx, M: Module>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+fn simd_pair_for_each_lane<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     x: CValue<'tcx>,
     y: CValue<'tcx>,
     ret: CPlace<'tcx>,
     f: impl Fn(
-        &mut FunctionCx<'_, 'tcx, M>,
+        &mut FunctionCx<'_, '_, 'tcx>,
         TyAndLayout<'tcx>,
         TyAndLayout<'tcx>,
         Value,
@@ -246,11 +215,11 @@ fn simd_pair_for_each_lane<'tcx, M: Module>(
     }
 }
 
-fn simd_reduce<'tcx, M: Module>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+fn simd_reduce<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value,
+    f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value,
 ) {
     let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     let lane_layout = fx.layout_of(lane_ty);
@@ -258,20 +227,19 @@ fn simd_reduce<'tcx, M: Module>(
 
     let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     for lane_idx in 1..lane_count {
-        let lane = val
-            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
-            .load_scalar(fx);
+        let lane =
+            val.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())).load_scalar(fx);
         res_val = f(fx, lane_layout, res_val, lane);
     }
     let res = CValue::by_val(res_val, lane_layout);
     ret.write_cvalue(fx, res);
 }
 
-fn simd_reduce_bool<'tcx, M: Module>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+fn simd_reduce_bool<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value,
+    f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
 ) {
     let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     assert!(ret.layout().ty.is_bool());
@@ -279,9 +247,8 @@ fn simd_reduce_bool<'tcx, M: Module>(
     let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
     for lane_idx in 1..lane_count {
-        let lane = val
-            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
-            .load_scalar(fx);
+        let lane =
+            val.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())).load_scalar(fx);
         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
         res_val = f(fx, res_val, lane);
     }
@@ -290,7 +257,7 @@ fn simd_reduce_bool<'tcx, M: Module>(
 }
 
 fn bool_to_zero_or_max_uint<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     val: Value,
 ) -> CValue<'tcx> {
@@ -424,7 +391,7 @@ macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
 }
 
 pub(crate) fn codegen_intrinsic_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     instance: Instance<'tcx>,
     args: &[mir::Operand<'tcx>],
     destination: Option<(CPlace<'tcx>, BasicBlock)>,
@@ -912,136 +879,175 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         };
 
         _ if intrinsic.starts_with("atomic_fence"), () {
-            crate::atomic_shim::lock_global_lock(fx);
-            crate::atomic_shim::unlock_global_lock(fx);
+            fx.bcx.ins().fence();
         };
         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {
-            crate::atomic_shim::lock_global_lock(fx);
-            crate::atomic_shim::unlock_global_lock(fx);
+            // FIXME use a compiler fence once Cranelift supports it
+            fx.bcx.ins().fence();
         };
-        _ if intrinsic.starts_with("atomic_load"), (c ptr) {
-            crate::atomic_shim::lock_global_lock(fx);
+        _ if intrinsic.starts_with("atomic_load"), <T> (v ptr) {
+            validate_atomic_type!(fx, intrinsic, span, T);
+            let ty = fx.clif_type(T).unwrap();
 
-            let inner_layout =
-                fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
-            validate_atomic_type!(fx, intrinsic, span, inner_layout.ty);
-            let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
-            ret.write_cvalue(fx, val);
+            let val = fx.bcx.ins().atomic_load(ty, MemFlags::trusted(), ptr);
 
-            crate::atomic_shim::unlock_global_lock(fx);
+            let val = CValue::by_val(val, fx.layout_of(T));
+            ret.write_cvalue(fx, val);
         };
         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
             validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
 
-            crate::atomic_shim::lock_global_lock(fx);
-
-            let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
-            dest.write_cvalue(fx, val);
+            let val = val.load_scalar(fx);
 
-            crate::atomic_shim::unlock_global_lock(fx);
+            fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
         };
-        _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, T);
-
-            crate::atomic_shim::lock_global_lock(fx);
+        _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) {
+            let layout = new.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
 
-            // Read old
-            let clif_ty = fx.clif_type(T).unwrap();
-            let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
-            ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
+            let new = new.load_scalar(fx);
 
-            // Write new
-            let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
-            dest.write_cvalue(fx, src);
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
 
-            crate::atomic_shim::unlock_global_lock(fx);
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
-            validate_atomic_type!(fx, intrinsic, span, T);
+        _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
+            let layout = new.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
 
             let test_old = test_old.load_scalar(fx);
             let new = new.load_scalar(fx);
 
-            crate::atomic_shim::lock_global_lock(fx);
-
-            // Read old
-            let clif_ty = fx.clif_type(T).unwrap();
-            let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
-
-            // Compare
+            let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
-            let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
-
-            // Write new
-            fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 
             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
-            ret.write_cvalue(fx, ret_val);
-
-            crate::atomic_shim::unlock_global_lock(fx);
+            ret.write_cvalue(fx, ret_val)
         };
 
-        _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) {
+            let layout = amount.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let amount = amount.load_scalar(fx);
-            atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) {
+            let layout = amount.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let amount = amount.load_scalar(fx);
-            atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, T);
+        _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
 
-            crate::atomic_shim::lock_global_lock(fx);
-
-            let clif_ty = fx.clif_type(T).unwrap();
-            let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
-            let and = fx.bcx.ins().band(old, src);
-            let new = fx.bcx.ins().bnot(and);
-            fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
-            ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
 
-            crate::atomic_shim::unlock_global_lock(fx);
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+
+        // FIXME https://github.com/bytecodealliance/wasmtime/issues/2647
+        _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
+        _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
 
-        _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
 
         minnumf32, (v a, v b) {
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index e0eb5c59590..27fc2abedc7 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -4,7 +4,7 @@ use super::*;
 use crate::prelude::*;
 
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     instance: Instance<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
@@ -85,10 +85,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 use rustc_middle::mir::interpret::*;
                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
 
-                let idx_bytes = match idx_const.val {
-                    ty::ConstKind::Value(ConstValue::ByRef { alloc, offset }) => {
+                let idx_bytes = match idx_const {
+                    ConstValue::ByRef { alloc, offset } => {
                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
-                        let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
+                        let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
                         alloc.get_bytes(fx, ptr, size).unwrap()
                     }
                     _ => unreachable!("{:?}", idx_const),
@@ -130,7 +130,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 );
             };
 
-            let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
@@ -159,7 +159,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 return;
             };
 
-            let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
@@ -276,5 +276,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         // simd_bitmask
         // simd_select
         // simd_rem
+        // simd_neg
+        // simd_trunc
+        // simd_floor
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index 1480ab25133..720d2a12534 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -1,18 +1,8 @@
-#![feature(
-    rustc_private,
-    decl_macro,
-    type_alias_impl_trait,
-    associated_type_bounds,
-    never_type,
-    try_blocks,
-    hash_drain_filter
-)]
+#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
 
-#[cfg(feature = "jit")]
-extern crate libc;
 extern crate snap;
 #[macro_use]
 extern crate rustc_middle;
@@ -53,12 +43,12 @@ mod abi;
 mod allocator;
 mod analyze;
 mod archive;
-mod atomic_shim;
 mod backend;
 mod base;
 mod cast;
 mod codegen_i128;
 mod common;
+mod compiler_builtins;
 mod constant;
 mod debuginfo;
 mod discriminant;
@@ -129,9 +119,9 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
     }
 }
 
-struct CodegenCx<'tcx, M: Module> {
+struct CodegenCx<'m, 'tcx: 'm> {
     tcx: TyCtxt<'tcx>,
-    module: M,
+    module: &'m mut dyn Module,
     global_asm: String,
     constants_cx: ConstantCx,
     cached_context: Context,
@@ -140,14 +130,20 @@ struct CodegenCx<'tcx, M: Module> {
     unwind_context: UnwindContext<'tcx>,
 }
 
-impl<'tcx, M: Module> CodegenCx<'tcx, M> {
-    fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self {
-        let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame);
-        let debug_context = if debug_info {
-            Some(DebugContext::new(tcx, module.isa()))
-        } else {
-            None
-        };
+impl<'m, 'tcx> CodegenCx<'m, 'tcx> {
+    fn new(
+        tcx: TyCtxt<'tcx>,
+        backend_config: BackendConfig,
+        module: &'m mut dyn Module,
+        debug_info: bool,
+    ) -> Self {
+        let unwind_context = UnwindContext::new(
+            tcx,
+            module.isa(),
+            matches!(backend_config.codegen_mode, CodegenMode::Aot),
+        );
+        let debug_context =
+            if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None };
         CodegenCx {
             tcx,
             module,
@@ -160,14 +156,9 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> {
         }
     }
 
-    fn finalize(mut self) -> (M, String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
-        self.constants_cx.finalize(self.tcx, &mut self.module);
-        (
-            self.module,
-            self.global_asm,
-            self.debug_context,
-            self.unwind_context,
-        )
+    fn finalize(self) -> (String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
+        self.constants_cx.finalize(self.tcx, self.module);
+        (self.global_asm, self.debug_context, self.unwind_context)
     }
 }
 
@@ -225,8 +216,10 @@ pub struct CraneliftCodegenBackend {
 
 impl CodegenBackend for CraneliftCodegenBackend {
     fn init(&self, sess: &Session) {
-        if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode {
-            sess.warn("LTO is not supported. You may get a linker error.");
+        use rustc_session::config::Lto;
+        match sess.lto() {
+            Lto::No | Lto::ThinLocal => {}
+            Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
         }
     }
 
@@ -241,9 +234,9 @@ impl CodegenBackend for CraneliftCodegenBackend {
         vec![]
     }
 
-    fn codegen_crate<'tcx>(
+    fn codegen_crate(
         &self,
-        tcx: TyCtxt<'tcx>,
+        tcx: TyCtxt<'_>,
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
@@ -253,9 +246,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                 .unwrap_or_else(|err| tcx.sess.fatal(&err))
         };
-        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
-
-        res
+        driver::codegen_crate(tcx, metadata, need_metadata_module, config)
     }
 
     fn join_codegen(
@@ -301,16 +292,9 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     let mut flags_builder = settings::builder();
     flags_builder.enable("is_pic").unwrap();
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
-    flags_builder
-        .set(
-            "enable_verifier",
-            if cfg!(debug_assertions) {
-                "true"
-            } else {
-                "false"
-            },
-        )
-        .unwrap();
+    let enable_verifier =
+        cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
+    flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();
 
     let tls_model = match target_triple.binary_format {
         BinaryFormat::Elf => "elf_gd",
@@ -322,27 +306,22 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
 
     flags_builder.set("enable_simd", "true").unwrap();
 
+    flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
+
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
             flags_builder.set("opt_level", "none").unwrap();
         }
         OptLevel::Less | OptLevel::Default => {}
-        OptLevel::Aggressive => {
+        OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
             flags_builder.set("opt_level", "speed_and_size").unwrap();
         }
-        OptLevel::Size | OptLevel::SizeMin => {
-            sess.warn("Optimizing for size is not supported. Just ignoring the request");
-        }
     }
 
     let flags = settings::Flags::new(flags_builder);
 
-    let variant = if cfg!(feature = "oldbe") {
-        cranelift_codegen::isa::BackendVariant::Legacy
-    } else {
-        cranelift_codegen::isa::BackendVariant::MachInst
-    };
+    let variant = cranelift_codegen::isa::BackendVariant::MachInst;
     let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
     // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt`
     // is interpreted as `bsr`.
diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs
index dc1e2107ce7..a564a59f725 100644
--- a/compiler/rustc_codegen_cranelift/src/linkage.rs
+++ b/compiler/rustc_codegen_cranelift/src/linkage.rs
@@ -6,8 +6,10 @@ pub(crate) fn get_clif_linkage(
     mono_item: MonoItem<'_>,
     linkage: RLinkage,
     visibility: Visibility,
+    is_compiler_builtins: bool,
 ) -> Linkage {
     match (linkage, visibility) {
+        (RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden,
         (RLinkage::External, Visibility::Default) => Linkage::Export,
         (RLinkage::Internal, Visibility::Default) => Linkage::Local,
         (RLinkage::External, Visibility::Hidden) => Linkage::Hidden,
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index b193cea877d..a6266f50776 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -1,3 +1,4 @@
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_hir::LangItem;
 use rustc_session::config::EntryFnType;
 
@@ -9,7 +10,6 @@ pub(crate) fn maybe_create_entry_wrapper(
     tcx: TyCtxt<'_>,
     module: &mut impl Module,
     unwind_context: &mut UnwindContext<'_>,
-    use_jit: bool,
 ) {
     let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) {
         Some((def_id, entry_ty)) => (
@@ -27,14 +27,7 @@ pub(crate) fn maybe_create_entry_wrapper(
         return;
     }
 
-    create_entry_fn(
-        tcx,
-        module,
-        unwind_context,
-        main_def_id,
-        use_start_lang_item,
-        use_jit,
-    );
+    create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item);
 
     fn create_entry_fn(
         tcx: TyCtxt<'_>,
@@ -42,7 +35,6 @@ pub(crate) fn maybe_create_entry_wrapper(
         unwind_context: &mut UnwindContext<'_>,
         rust_main_def_id: DefId,
         use_start_lang_item: bool,
-        use_jit: bool,
     ) {
         let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
         // Given that `main()` has no arguments,
@@ -57,23 +49,17 @@ pub(crate) fn maybe_create_entry_wrapper(
                 AbiParam::new(m.target_config().pointer_type()),
                 AbiParam::new(m.target_config().pointer_type()),
             ],
-            returns: vec![AbiParam::new(
-                m.target_config().pointer_type(), /*isize*/
-            )],
+            returns: vec![AbiParam::new(m.target_config().pointer_type() /*isize*/)],
             call_conv: CallConv::triple_default(m.isa().triple()),
         };
 
-        let cmain_func_id = m
-            .declare_function("main", Linkage::Export, &cmain_sig)
-            .unwrap();
+        let cmain_func_id = m.declare_function("main", Linkage::Export, &cmain_sig).unwrap();
 
         let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
 
         let main_name = tcx.symbol_name(instance).name.to_string();
         let main_sig = get_function_sig(tcx, m.isa().triple(), instance);
-        let main_func_id = m
-            .declare_function(&main_name, Linkage::Import, &main_sig)
-            .unwrap();
+        let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap();
 
         let mut ctx = Context::new();
         ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig);
@@ -86,8 +72,6 @@ pub(crate) fn maybe_create_entry_wrapper(
             let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type());
             let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type());
 
-            crate::atomic_shim::init_global_lock(m, &mut bcx, use_jit);
-
             let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
 
             let call_inst = if use_start_lang_item {
@@ -103,9 +87,7 @@ pub(crate) fn maybe_create_entry_wrapper(
                 .polymorphize(tcx);
                 let start_func_id = import_function(tcx, m, start_instance);
 
-                let main_val = bcx
-                    .ins()
-                    .func_addr(m.target_config().pointer_type(), main_func_ref);
+                let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
 
                 let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
                 bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv])
@@ -119,12 +101,8 @@ pub(crate) fn maybe_create_entry_wrapper(
             bcx.seal_all_blocks();
             bcx.finalize();
         }
-        m.define_function(
-            cmain_func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
-        .unwrap();
+        m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
+            .unwrap();
         unwind_context.add_function(cmain_func_id, &ctx, m.isa());
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/metadata.rs b/compiler/rustc_codegen_cranelift/src/metadata.rs
index 2e3b9fb8364..dbdc8cbad44 100644
--- a/compiler/rustc_codegen_cranelift/src/metadata.rs
+++ b/compiler/rustc_codegen_cranelift/src/metadata.rs
@@ -1,10 +1,10 @@
 //! Reading and writing of the rustc metadata for rlibs and dylibs
 
-use std::convert::TryFrom;
 use std::fs::File;
 use std::path::Path;
 
 use rustc_codegen_ssa::METADATA_FILENAME;
+use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::owning_ref::OwningRef;
 use rustc_data_structures::rustc_erase_owner;
 use rustc_data_structures::sync::MetadataRef;
@@ -17,38 +17,43 @@ use crate::backend::WriteMetadata;
 
 pub(crate) struct CraneliftMetadataLoader;
 
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<MetadataRef, String> {
+    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
+    let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?;
+    let metadata = OwningRef::new(data).try_map(f)?;
+    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+}
+
 impl MetadataLoader for CraneliftMetadataLoader {
     fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
-        // Iterate over all entries in the archive:
-        while let Some(entry_result) = archive.next_entry() {
-            let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
-            if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
-                let mut buf = Vec::with_capacity(
-                    usize::try_from(entry.header().size())
-                        .expect("Rlib metadata file too big to load into memory."),
-                );
-                ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?;
-                let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-                return Ok(rustc_erase_owner!(buf.map_owner_box()));
+        load_metadata_with(path, |data| {
+            let archive = object::read::archive::ArchiveFile::parse(&*data)
+                .map_err(|e| format!("{:?}", e))?;
+
+            for entry_result in archive.members() {
+                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
+                if entry.name() == METADATA_FILENAME.as_bytes() {
+                    return Ok(entry.data());
+                }
             }
-        }
 
-        Err("couldn't find metadata entry".to_string())
+            Err("couldn't find metadata entry".to_string())
+        })
     }
 
     fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
         use object::{Object, ObjectSection};
-        let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
-        let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
-        let buf = file
-            .section_by_name(".rustc")
-            .ok_or("no .rustc section")?
-            .data()
-            .map_err(|e| format!("failed to read .rustc section: {:?}", e))?
-            .to_owned();
-        let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-        Ok(rustc_erase_owner!(buf.map_owner_box()))
+
+        load_metadata_with(path, |data| {
+            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
+            file.section_by_name(".rustc")
+                .ok_or("no .rustc section")?
+                .data()
+                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
+        })
     }
 }
 
@@ -94,9 +99,7 @@ pub(crate) fn write_metadata<P: WriteMetadata>(
 
     assert!(kind == MetadataKind::Compressed);
     let mut compressed = tcx.metadata_encoding_version();
-    FrameEncoder::new(&mut compressed)
-        .write_all(&metadata.raw_data)
-        .unwrap();
+    FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
 
     product.add_rustc_section(
         rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx),
diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs
index d1d2b3b872a..2ebf30da2d8 100644
--- a/compiler/rustc_codegen_cranelift/src/num.rs
+++ b/compiler/rustc_codegen_cranelift/src/num.rs
@@ -41,7 +41,7 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
 }
 
 fn codegen_compare_bin_op<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     signed: bool,
     lhs: Value,
@@ -54,7 +54,7 @@ fn codegen_compare_bin_op<'tcx>(
 }
 
 pub(crate) fn codegen_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -93,17 +93,12 @@ pub(crate) fn codegen_binop<'tcx>(
         ty::Uint(_) | ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs),
         ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs),
         ty::RawPtr(..) | ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs),
-        _ => unreachable!(
-            "{:?}({:?}, {:?})",
-            bin_op,
-            in_lhs.layout().ty,
-            in_rhs.layout().ty
-        ),
+        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
     }
 }
 
 pub(crate) fn codegen_bool_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -124,7 +119,7 @@ pub(crate) fn codegen_bool_binop<'tcx>(
 }
 
 pub(crate) fn codegen_int_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -171,13 +166,11 @@ pub(crate) fn codegen_int_binop<'tcx>(
         BinOp::Shl => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             fx.bcx.ins().ishl(lhs, actual_shift)
         }
         BinOp::Shr => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             if signed {
                 fx.bcx.ins().sshr(lhs, actual_shift)
             } else {
@@ -185,19 +178,14 @@ pub(crate) fn codegen_int_binop<'tcx>(
             }
         }
         // Compare binops handles by `codegen_binop`.
-        _ => unreachable!(
-            "{:?}({:?}, {:?})",
-            bin_op,
-            in_lhs.layout().ty,
-            in_rhs.layout().ty
-        ),
+        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
     };
 
     CValue::by_val(val, in_lhs.layout())
 }
 
 pub(crate) fn codegen_checked_int_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -268,9 +256,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
                     let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs);
                     let val = fx.bcx.ins().imul(lhs, rhs);
                     let has_underflow =
-                        fx.bcx
-                            .ins()
-                            .icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
+                        fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
                     let has_overflow = fx.bcx.ins().icmp_imm(
                         IntCC::SignedGreaterThan,
                         val,
@@ -309,10 +295,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
             let val = fx.bcx.ins().ishl(lhs, actual_shift);
             let ty = fx.bcx.func.dfg.value_type(val);
             let max_shift = i64::from(ty.bits()) - 1;
-            let has_overflow = fx
-                .bcx
-                .ins()
-                .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
+            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
             (val, has_overflow)
         }
         BinOp::Shr => {
@@ -326,38 +309,20 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
             };
             let ty = fx.bcx.func.dfg.value_type(val);
             let max_shift = i64::from(ty.bits()) - 1;
-            let has_overflow = fx
-                .bcx
-                .ins()
-                .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
+            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
             (val, has_overflow)
         }
-        _ => bug!(
-            "binop {:?} on checked int/uint lhs: {:?} rhs: {:?}",
-            bin_op,
-            in_lhs,
-            in_rhs
-        ),
+        _ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
     };
 
     let has_overflow = fx.bcx.ins().bint(types::I8, has_overflow);
 
-    // FIXME directly write to result place instead
-    let out_place = CPlace::new_stack_slot(
-        fx,
-        fx.layout_of(
-            fx.tcx
-                .mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()),
-        ),
-    );
-    let out_layout = out_place.layout();
-    out_place.write_cvalue(fx, CValue::by_val_pair(res, has_overflow, out_layout));
-
-    out_place.to_cvalue(fx)
+    let out_layout = fx.layout_of(fx.tcx.mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()));
+    CValue::by_val_pair(res, has_overflow, out_layout)
 }
 
 pub(crate) fn codegen_float_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -402,7 +367,7 @@ pub(crate) fn codegen_float_binop<'tcx>(
 }
 
 pub(crate) fn codegen_ptr_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -420,7 +385,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let lhs = in_lhs.load_scalar(fx);
                 let rhs = in_rhs.load_scalar(fx);
 
-                return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs);
+                codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
             }
             BinOp::Offset => {
                 let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
@@ -429,10 +394,10 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
                 let base_val = base.load_scalar(fx);
                 let res = fx.bcx.ins().iadd(base_val, ptr_diff);
-                return CValue::by_val(res, base.layout());
+                CValue::by_val(res, base.layout())
             }
             _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
-        };
+        }
     } else {
         let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
         let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);
@@ -452,9 +417,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
 
                 let ptr_cmp =
-                    fx.bcx
-                        .ins()
-                        .icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
+                    fx.bcx.ins().icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
                 let extra_cmp = fx.bcx.ins().icmp(
                     bin_op_to_intcc(bin_op, false).unwrap(),
                     lhs_extra,
@@ -466,9 +429,6 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
             _ => panic!("bin_op {:?} on ptr", bin_op),
         };
 
-        CValue::by_val(
-            fx.bcx.ins().bint(types::I8, res),
-            fx.layout_of(fx.tcx.types.bool),
-        )
+        CValue::by_val(fx.bcx.ins().bint(types::I8, res), fx.layout_of(fx.tcx.types.bool))
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs b/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs
index f02732014d1..ca9ff15ec10 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs
@@ -15,10 +15,7 @@ pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block
     // bytecodealliance/cranelift#1339 is implemented.
 
     let mut block_insts = FxHashMap::default();
-    for block in cold_blocks
-        .keys()
-        .filter(|&block| cold_blocks.contains(block))
-    {
+    for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
         let insts = ctx.func.layout.block_insts(block).collect::<Vec<_>>();
         for &inst in &insts {
             ctx.func.layout.remove_inst(inst);
@@ -28,10 +25,7 @@ pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block
     }
 
     // And then append them at the back again.
-    for block in cold_blocks
-        .keys()
-        .filter(|&block| cold_blocks.contains(block))
-    {
+    for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
         ctx.func.layout.append_block(block);
         for inst in block_insts.remove(&block).unwrap() {
             ctx.func.layout.append_inst(inst, block);
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
index 3ce7f8cd9a8..389f50e797e 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
@@ -19,7 +19,12 @@ pub(crate) fn optimize_function<'tcx>(
     if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No {
         return; // FIXME classify optimizations over opt levels
     }
-    self::stack2reg::optimize_function(ctx, clif_comments);
+
+    // FIXME(#1142) stack2reg miscompiles lewton
+    if false {
+        self::stack2reg::optimize_function(ctx, clif_comments);
+    }
+
     crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments);
     crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
 }
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
index a575ed8dc35..b95e2d72877 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
@@ -10,10 +10,7 @@ use cranelift_frontend::FunctionBuilder;
 pub(crate) fn maybe_unwrap_bint(bcx: &mut FunctionBuilder<'_>, arg: Value) -> Value {
     if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
         match bcx.func.dfg[arg_inst] {
-            InstructionData::Unary {
-                opcode: Opcode::Bint,
-                arg,
-            } => arg,
+            InstructionData::Unary { opcode: Opcode::Bint, arg } => arg,
             _ => arg,
         }
     } else {
@@ -54,12 +51,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -
 
         match bcx.func.dfg[arg_inst] {
             // This is the lowering of Rvalue::Not
-            InstructionData::Load {
-                opcode: Opcode::Load,
-                arg: ptr,
-                flags,
-                offset,
-            } => {
+            InstructionData::Load { opcode: Opcode::Load, arg: ptr, flags, offset } => {
                 // Using `load.i8 + uextend.i32` would legalize to `uload8 + ireduce.i8 +
                 // uextend.i32`. Just `uload8` is much faster.
                 match bcx.func.dfg.ctrl_typevar(arg_inst) {
@@ -95,20 +87,14 @@ pub(crate) fn maybe_known_branch_taken(
     };
 
     match bcx.func.dfg[arg_inst] {
-        InstructionData::UnaryBool {
-            opcode: Opcode::Bconst,
-            imm,
-        } => {
+        InstructionData::UnaryBool { opcode: Opcode::Bconst, imm } => {
             if test_zero {
                 Some(!imm)
             } else {
                 Some(imm)
             }
         }
-        InstructionData::UnaryImm {
-            opcode: Opcode::Iconst,
-            imm,
-        } => {
+        InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => {
             if test_zero {
                 Some(imm.bits() == 0)
             } else {
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
index 3c939d5a586..8bb02a3e558 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
@@ -175,16 +175,13 @@ impl<'a> OptimizeContext<'a> {
             }
         }
 
-        OptimizeContext {
-            ctx,
-            stack_slot_usage_map,
-        }
+        OptimizeContext { ctx, stack_slot_usage_map }
     }
 }
 
 pub(super) fn optimize_function(
     ctx: &mut Context,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] clif_comments: &mut crate::pretty_clif::CommentWriter,
+    clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
     combine_stack_addr_with_load_store(&mut ctx.func);
 
@@ -194,8 +191,7 @@ pub(super) fn optimize_function(
 
     remove_unused_stack_addr_and_stack_load(&mut opt_ctx);
 
-    #[cfg(debug_assertions)]
-    {
+    if clif_comments.enabled() {
         for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
             clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
         }
@@ -211,25 +207,27 @@ pub(super) fn optimize_function(
         for load in users.stack_load.clone().into_iter() {
             let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);
 
-            #[cfg(debug_assertions)]
-            for &store in &potential_stores {
-                clif_comments.add_comment(
-                    load,
-                    format!(
-                        "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &store in &potential_stores {
+                    clif_comments.add_comment(
+                        load,
+                        format!(
+                            "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
             }
 
             match *potential_stores {
                 [] => {
-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    if clif_comments.enabled() {
+                        clif_comments
+                            .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    }
                 }
                 [store]
                     if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
@@ -239,9 +237,12 @@ pub(super) fn optimize_function(
                     // Only one store could have been the origin of the value.
                     let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];
 
-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, format!("Store to load forward {} -> {}", store, load));
+                    if clif_comments.enabled() {
+                        clif_comments.add_comment(
+                            load,
+                            format!("Store to load forward {} -> {}", store, load),
+                        );
+                    }
 
                     users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
                 }
@@ -252,33 +253,35 @@ pub(super) fn optimize_function(
         for store in users.stack_store.clone().into_iter() {
             let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);
 
-            #[cfg(debug_assertions)]
-            for &load in &potential_loads {
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Potential load from store {} <- {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &load in &potential_loads {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Potential load from store {} <- {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
             }
 
             if potential_loads.is_empty() {
                 // Never loaded; can safely remove all stores and the stack slot.
                 // FIXME also remove stores when there is always a next store before a load.
 
-                #[cfg(debug_assertions)]
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Remove dead stack store {} of {}",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        stack_slot.0
-                    ),
-                );
+                if clif_comments.enabled() {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Remove dead stack store {} of {}",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            stack_slot.0
+                        ),
+                    );
+                }
 
                 users.remove_dead_store(&mut opt_ctx.ctx.func, store);
             }
@@ -296,12 +299,7 @@ fn combine_stack_addr_with_load_store(func: &mut Function) {
     while let Some(_block) = cursor.next_block() {
         while let Some(inst) = cursor.next_inst() {
             match cursor.func.dfg[inst] {
-                InstructionData::Load {
-                    opcode: Opcode::Load,
-                    arg: addr,
-                    flags: _,
-                    offset,
-                } => {
+                InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => {
                     if cursor.func.dfg.ctrl_typevar(inst) == types::I128
                         || cursor.func.dfg.ctrl_typevar(inst).is_vector()
                     {
@@ -391,20 +389,14 @@ fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) {
         stack_slot_users
             .stack_addr
             .drain_filter(|inst| {
-                stack_addr_load_insts_users
-                    .get(inst)
-                    .map(|users| users.is_empty())
-                    .unwrap_or(true)
+                stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
             })
             .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst));
 
         stack_slot_users
             .stack_load
             .drain_filter(|inst| {
-                stack_addr_load_insts_users
-                    .get(inst)
-                    .map(|users| users.is_empty())
-                    .unwrap_or(true)
+                stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
             })
             .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst));
     }
@@ -415,11 +407,8 @@ fn try_get_stack_slot_and_offset_for_addr(
     addr: Value,
 ) -> Option<(StackSlot, Offset32)> {
     if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) {
-        if let InstructionData::StackLoad {
-            opcode: Opcode::StackAddr,
-            stack_slot,
-            offset,
-        } = func.dfg[addr_inst]
+        if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } =
+            func.dfg[addr_inst]
         {
             return Some((stack_slot, offset));
         }
@@ -437,16 +426,8 @@ enum SpatialOverlap {
 fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
     fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) {
         match func.dfg[inst] {
-            InstructionData::StackLoad {
-                opcode: Opcode::StackAddr,
-                stack_slot,
-                offset,
-            }
-            | InstructionData::StackLoad {
-                opcode: Opcode::StackLoad,
-                stack_slot,
-                offset,
-            }
+            InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset }
+            | InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset }
             | InstructionData::StackStore {
                 opcode: Opcode::StackStore,
                 stack_slot,
@@ -471,10 +452,7 @@ fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
     }
 
     let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into();
-    let dest_end: i64 = dest_offset
-        .try_add_i64(i64::from(dest_size))
-        .unwrap()
-        .into();
+    let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into();
     if src_end <= dest_offset.into() || dest_end <= src_offset.into() {
         return SpatialOverlap::No;
     }
diff --git a/compiler/rustc_codegen_cranelift/src/pointer.rs b/compiler/rustc_codegen_cranelift/src/pointer.rs
index b2036d7bcd4..31d827f83bf 100644
--- a/compiler/rustc_codegen_cranelift/src/pointer.rs
+++ b/compiler/rustc_codegen_cranelift/src/pointer.rs
@@ -23,82 +23,48 @@ pub(crate) enum PointerBase {
 
 impl Pointer {
     pub(crate) fn new(addr: Value) -> Self {
-        Pointer {
-            base: PointerBase::Addr(addr),
-            offset: Offset32::new(0),
-        }
+        Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
     }
 
     pub(crate) fn stack_slot(stack_slot: StackSlot) -> Self {
-        Pointer {
-            base: PointerBase::Stack(stack_slot),
-            offset: Offset32::new(0),
-        }
+        Pointer { base: PointerBase::Stack(stack_slot), offset: Offset32::new(0) }
     }
 
-    pub(crate) fn const_addr<'a, 'tcx>(
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        addr: i64,
-    ) -> Self {
+    pub(crate) fn const_addr(fx: &mut FunctionCx<'_, '_, '_>, addr: i64) -> Self {
         let addr = fx.bcx.ins().iconst(fx.pointer_type, addr);
-        Pointer {
-            base: PointerBase::Addr(addr),
-            offset: Offset32::new(0),
-        }
+        Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
     }
 
     pub(crate) fn dangling(align: Align) -> Self {
-        Pointer {
-            base: PointerBase::Dangling(align),
-            offset: Offset32::new(0),
-        }
+        Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
     }
 
-    #[cfg(debug_assertions)]
-    pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) {
+    pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) {
         (self.base, self.offset)
     }
 
-    pub(crate) fn get_addr<'a, 'tcx>(self, fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> Value {
+    pub(crate) fn get_addr(self, fx: &mut FunctionCx<'_, '_, '_>) -> Value {
         match self.base {
             PointerBase::Addr(base_addr) => {
                 let offset: i64 = self.offset.into();
-                if offset == 0 {
-                    base_addr
-                } else {
-                    fx.bcx.ins().iadd_imm(base_addr, offset)
-                }
+                if offset == 0 { base_addr } else { fx.bcx.ins().iadd_imm(base_addr, offset) }
             }
             PointerBase::Stack(stack_slot) => {
-                fx.bcx
-                    .ins()
-                    .stack_addr(fx.pointer_type, stack_slot, self.offset)
+                fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset)
+            }
+            PointerBase::Dangling(align) => {
+                fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap())
             }
-            PointerBase::Dangling(align) => fx
-                .bcx
-                .ins()
-                .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()),
         }
     }
 
-    pub(crate) fn offset<'a, 'tcx>(
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        extra_offset: Offset32,
-    ) -> Self {
+    pub(crate) fn offset(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Offset32) -> Self {
         self.offset_i64(fx, extra_offset.into())
     }
 
-    pub(crate) fn offset_i64<'a, 'tcx>(
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        extra_offset: i64,
-    ) -> Self {
+    pub(crate) fn offset_i64(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: i64) -> Self {
         if let Some(new_offset) = self.offset.try_add_i64(extra_offset) {
-            Pointer {
-                base: self.base,
-                offset: new_offset,
-            }
+            Pointer { base: self.base, offset: new_offset }
         } else {
             let base_offset: i64 = self.offset.into();
             if let Some(new_offset) = base_offset.checked_add(extra_offset) {
@@ -107,16 +73,12 @@ impl Pointer {
                     PointerBase::Stack(stack_slot) => {
                         fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0)
                     }
-                    PointerBase::Dangling(align) => fx
-                        .bcx
-                        .ins()
-                        .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()),
+                    PointerBase::Dangling(align) => {
+                        fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap())
+                    }
                 };
                 let addr = fx.bcx.ins().iadd_imm(base_addr, new_offset);
-                Pointer {
-                    base: PointerBase::Addr(addr),
-                    offset: Offset32::new(0),
-                }
+                Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
             } else {
                 panic!(
                     "self.offset ({}) + extra_offset ({}) not representable in i64",
@@ -126,31 +88,22 @@ impl Pointer {
         }
     }
 
-    pub(crate) fn offset_value<'a, 'tcx>(
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        extra_offset: Value,
-    ) -> Self {
+    pub(crate) fn offset_value(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Value) -> Self {
         match self.base {
             PointerBase::Addr(addr) => Pointer {
                 base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)),
                 offset: self.offset,
             },
             PointerBase::Stack(stack_slot) => {
-                let base_addr = fx
-                    .bcx
-                    .ins()
-                    .stack_addr(fx.pointer_type, stack_slot, self.offset);
+                let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset);
                 Pointer {
                     base: PointerBase::Addr(fx.bcx.ins().iadd(base_addr, extra_offset)),
                     offset: Offset32::new(0),
                 }
             }
             PointerBase::Dangling(align) => {
-                let addr = fx
-                    .bcx
-                    .ins()
-                    .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap());
+                let addr =
+                    fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap());
                 Pointer {
                     base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)),
                     offset: self.offset,
@@ -159,46 +112,21 @@ impl Pointer {
         }
     }
 
-    pub(crate) fn load<'a, 'tcx>(
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        ty: Type,
-        flags: MemFlags,
-    ) -> Value {
+    pub(crate) fn load(self, fx: &mut FunctionCx<'_, '_, '_>, ty: Type, flags: MemFlags) -> Value {
         match self.base {
             PointerBase::Addr(base_addr) => fx.bcx.ins().load(ty, flags, base_addr, self.offset),
-            PointerBase::Stack(stack_slot) => {
-                if ty == types::I128 || ty.is_vector() {
-                    // WORKAROUND for stack_load.i128 and stack_load.iXxY not being implemented
-                    let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
-                    fx.bcx.ins().load(ty, flags, base_addr, self.offset)
-                } else {
-                    fx.bcx.ins().stack_load(ty, stack_slot, self.offset)
-                }
-            }
+            PointerBase::Stack(stack_slot) => fx.bcx.ins().stack_load(ty, stack_slot, self.offset),
             PointerBase::Dangling(_align) => unreachable!(),
         }
     }
 
-    pub(crate) fn store<'a, 'tcx>(
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        value: Value,
-        flags: MemFlags,
-    ) {
+    pub(crate) fn store(self, fx: &mut FunctionCx<'_, '_, '_>, value: Value, flags: MemFlags) {
         match self.base {
             PointerBase::Addr(base_addr) => {
                 fx.bcx.ins().store(flags, value, base_addr, self.offset);
             }
             PointerBase::Stack(stack_slot) => {
-                let val_ty = fx.bcx.func.dfg.value_type(value);
-                if val_ty == types::I128 || val_ty.is_vector() {
-                    // WORKAROUND for stack_store.i128 and stack_store.iXxY not being implemented
-                    let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
-                    fx.bcx.ins().store(flags, value, base_addr, self.offset);
-                } else {
-                    fx.bcx.ins().stack_store(value, stack_slot, self.offset);
-                }
+                fx.bcx.ins().stack_store(value, stack_slot, self.offset);
             }
             PointerBase::Dangling(_align) => unreachable!(),
         }
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index f4a15ab12d5..d22ea3772ee 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -69,36 +69,36 @@ use crate::prelude::*;
 
 #[derive(Debug)]
 pub(crate) struct CommentWriter {
+    enabled: bool,
     global_comments: Vec<String>,
     entity_comments: FxHashMap<AnyEntity, String>,
 }
 
 impl CommentWriter {
     pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
-        let global_comments = if cfg!(debug_assertions) {
+        let enabled = should_write_ir(tcx);
+        let global_comments = if enabled {
             vec![
                 format!("symbol {}", tcx.symbol_name(instance).name),
                 format!("instance {:?}", instance),
-                format!(
-                    "abi {:?}",
-                    FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])
-                ),
+                format!("abi {:?}", FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])),
                 String::new(),
             ]
         } else {
             vec![]
         };
 
-        CommentWriter {
-            global_comments,
-            entity_comments: FxHashMap::default(),
-        }
+        CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
     }
 }
 
-#[cfg(debug_assertions)]
 impl CommentWriter {
+    pub(crate) fn enabled(&self) -> bool {
+        self.enabled
+    }
+
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
+        debug_assert!(self.enabled);
         self.global_comments.push(comment.into());
     }
 
@@ -107,6 +107,8 @@ impl CommentWriter {
         entity: E,
         comment: S,
     ) {
+        debug_assert!(self.enabled);
+
         use std::collections::hash_map::Entry;
         match self.entity_comments.entry(entity.into()) {
             Entry::Occupied(mut occ) => {
@@ -185,8 +187,7 @@ impl FuncWriter for &'_ CommentWriter {
     }
 }
 
-#[cfg(debug_assertions)]
-impl<M: Module> FunctionCx<'_, '_, M> {
+impl FunctionCx<'_, '_, '_> {
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
         self.clif_comments.add_global_comment(comment);
     }
@@ -201,16 +202,11 @@ impl<M: Module> FunctionCx<'_, '_, M> {
 }
 
 pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
-    cfg!(debug_assertions)
-        || tcx
-            .sess
-            .opts
-            .output_types
-            .contains_key(&OutputType::LlvmAssembly)
+    tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
 }
 
-pub(crate) fn write_ir_file<'tcx>(
-    tcx: TyCtxt<'tcx>,
+pub(crate) fn write_ir_file(
+    tcx: TyCtxt<'_>,
     name: &str,
     write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
 ) {
@@ -228,10 +224,7 @@ pub(crate) fn write_ir_file<'tcx>(
 
     let clif_file_name = clif_output_dir.join(name);
 
-    let res: std::io::Result<()> = try {
-        let mut file = std::fs::File::create(clif_file_name)?;
-        write(&mut file)?;
-    };
+    let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
     if let Err(err) = res {
         tcx.sess.warn(&format!("error writing ir file: {}", err));
     }
@@ -245,40 +238,33 @@ pub(crate) fn write_clif_file<'tcx>(
     context: &cranelift_codegen::Context,
     mut clif_comments: &CommentWriter,
 ) {
-    write_ir_file(
-        tcx,
-        &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
-        |file| {
-            let value_ranges = isa.map(|isa| {
-                context
-                    .build_value_labels_ranges(isa)
-                    .expect("value location ranges")
-            });
+    write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| {
+        let value_ranges =
+            isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
 
-            let mut clif = String::new();
-            cranelift_codegen::write::decorate_function(
-                &mut clif_comments,
-                &mut clif,
-                &context.func,
-                &DisplayFunctionAnnotations {
-                    isa: Some(&*crate::build_isa(tcx.sess)),
-                    value_ranges: value_ranges.as_ref(),
-                },
-            )
-            .unwrap();
+        let mut clif = String::new();
+        cranelift_codegen::write::decorate_function(
+            &mut clif_comments,
+            &mut clif,
+            &context.func,
+            &DisplayFunctionAnnotations {
+                isa: Some(&*crate::build_isa(tcx.sess)),
+                value_ranges: value_ranges.as_ref(),
+            },
+        )
+        .unwrap();
 
-            writeln!(file, "test compile")?;
-            writeln!(file, "set is_pic")?;
-            writeln!(file, "set enable_simd")?;
-            writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
-            writeln!(file)?;
-            file.write_all(clif.as_bytes())?;
-            Ok(())
-        },
-    );
+        writeln!(file, "test compile")?;
+        writeln!(file, "set is_pic")?;
+        writeln!(file, "set enable_simd")?;
+        writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
+        writeln!(file)?;
+        file.write_all(clif.as_bytes())?;
+        Ok(())
+    });
 }
 
-impl<M: Module> fmt::Debug for FunctionCx<'_, '_, M> {
+impl fmt::Debug for FunctionCx<'_, '_, '_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "{:?}", self.instance.substs)?;
         writeln!(f, "{:?}", self.local_map)?;
diff --git a/compiler/rustc_codegen_cranelift/src/toolchain.rs b/compiler/rustc_codegen_cranelift/src/toolchain.rs
index 735c59d70c1..484a9b699a0 100644
--- a/compiler/rustc_codegen_cranelift/src/toolchain.rs
+++ b/compiler/rustc_codegen_cranelift/src/toolchain.rs
@@ -71,12 +71,9 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
                 flavor,
             )),
             (Some(linker), None) => {
-                let stem = linker
-                    .file_stem()
-                    .and_then(|stem| stem.to_str())
-                    .unwrap_or_else(|| {
-                        sess.fatal("couldn't extract file stem from specified linker")
-                    });
+                let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| {
+                    sess.fatal("couldn't extract file stem from specified linker")
+                });
 
                 let flavor = if stem == "emcc" {
                     LinkerFlavor::Em
@@ -105,11 +102,7 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
 
     // linker and linker flavor specified via command line have precedence over what the target
     // specification specifies
-    if let Some(ret) = infer_from(
-        sess,
-        sess.opts.cg.linker.clone(),
-        sess.opts.cg.linker_flavor,
-    ) {
+    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
         return ret;
     }
 
diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs
index 67495c74148..1ab0703e981 100644
--- a/compiler/rustc_codegen_cranelift/src/trap.rs
+++ b/compiler/rustc_codegen_cranelift/src/trap.rs
@@ -2,7 +2,7 @@
 
 use crate::prelude::*;
 
-fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) {
+fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
     let puts = fx
         .cx
         .module
@@ -17,8 +17,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) {
         )
         .unwrap();
     let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    {
+    if fx.clif_comments.enabled() {
         fx.add_comment(puts, "puts");
     }
 
@@ -29,7 +28,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) {
 }
 
 /// Trap code: user1
-pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
+pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
     codegen_print(fx, msg.as_ref());
     fx.bcx.ins().trap(TrapCode::User(1));
 }
@@ -38,7 +37,7 @@ pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsR
 /// so you can **not** add instructions to it afterwards.
 ///
 /// Trap code: user65535
-pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
+pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
     codegen_print(fx, msg.as_ref());
     fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
 }
@@ -47,7 +46,7 @@ pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, impl Module>, msg: im
 ///
 /// Trap code: user65535
 pub(crate) fn trap_unreachable_ret_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     dest_layout: TyAndLayout<'tcx>,
     msg: impl AsRef<str>,
 ) -> CValue<'tcx> {
@@ -62,7 +61,7 @@ pub(crate) fn trap_unreachable_ret_value<'tcx>(
 /// to it afterwards.
 ///
 /// Trap code: user65535
-pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
+pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
     codegen_print(fx, msg.as_ref());
     let true_ = fx.bcx.ins().iconst(types::I32, 1);
     fx.bcx.ins().trapnz(true_, TrapCode::User(!0));
@@ -72,7 +71,7 @@ pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg:
 ///
 /// Trap code: user65535
 pub(crate) fn trap_unimplemented_ret_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     dest_layout: TyAndLayout<'tcx>,
     msg: impl AsRef<str>,
 ) -> CValue<'tcx> {
diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs
index c77ff5d56ba..042583cd572 100644
--- a/compiler/rustc_codegen_cranelift/src/unsize.rs
+++ b/compiler/rustc_codegen_cranelift/src/unsize.rs
@@ -13,19 +13,18 @@ use crate::prelude::*;
 /// in an upcast, where the new vtable for an object will be derived
 /// from the old one.
 pub(crate) fn unsized_info<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     source: Ty<'tcx>,
     target: Ty<'tcx>,
     old_info: Option<Value>,
 ) -> Value {
     let (source, target) =
-        fx.tcx
-            .struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all());
+        fx.tcx.struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all());
     match (&source.kind(), &target.kind()) {
-        (&ty::Array(_, len), &ty::Slice(_)) => fx.bcx.ins().iconst(
-            fx.pointer_type,
-            len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64,
-        ),
+        (&ty::Array(_, len), &ty::Slice(_)) => fx
+            .bcx
+            .ins()
+            .iconst(fx.pointer_type, len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64),
         (&ty::Dynamic(..), &ty::Dynamic(..)) => {
             // For now, upcasts are limited to changes in marker
             // traits, and hence never actually require an actual
@@ -35,17 +34,13 @@ pub(crate) fn unsized_info<'tcx>(
         (_, &ty::Dynamic(ref data, ..)) => {
             crate::vtable::get_vtable(fx, fx.layout_of(source), data.principal())
         }
-        _ => bug!(
-            "unsized_info: invalid unsizing {:?} -> {:?}",
-            source,
-            target
-        ),
+        _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
     }
 }
 
 /// Coerce `src` to `dst_ty`. `src_ty` must be a thin pointer.
 fn unsize_thin_ptr<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     src: Value,
     src_layout: TyAndLayout<'tcx>,
     dst_layout: TyAndLayout<'tcx>,
@@ -89,24 +84,22 @@ fn unsize_thin_ptr<'tcx>(
 /// Coerce `src`, which is a reference to a value of type `src_ty`,
 /// to a value of type `dst_ty` and store the result in `dst`
 pub(crate) fn coerce_unsized_into<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     src: CValue<'tcx>,
     dst: CPlace<'tcx>,
 ) {
     let src_ty = src.layout().ty;
     let dst_ty = dst.layout().ty;
     let mut coerce_ptr = || {
-        let (base, info) = if fx
-            .layout_of(src.layout().ty.builtin_deref(true).unwrap().ty)
-            .is_unsized()
-        {
-            // fat-ptr to fat-ptr unsize preserves the vtable
-            // i.e., &'a fmt::Debug+Send => &'a fmt::Debug
-            src.load_scalar_pair(fx)
-        } else {
-            let base = src.load_scalar(fx);
-            unsize_thin_ptr(fx, base, src.layout(), dst.layout())
-        };
+        let (base, info) =
+            if fx.layout_of(src.layout().ty.builtin_deref(true).unwrap().ty).is_unsized() {
+                // fat-ptr to fat-ptr unsize preserves the vtable
+                // i.e., &'a fmt::Debug+Send => &'a fmt::Debug
+                src.load_scalar_pair(fx)
+            } else {
+                let base = src.load_scalar(fx);
+                unsize_thin_ptr(fx, base, src.layout(), dst.layout())
+            };
         dst.write_cvalue(fx, CValue::by_val_pair(base, info, dst.layout()));
     };
     match (&src_ty.kind(), &dst_ty.kind()) {
@@ -131,39 +124,26 @@ pub(crate) fn coerce_unsized_into<'tcx>(
                 }
             }
         }
-        _ => bug!(
-            "coerce_unsized_into: invalid coercion {:?} -> {:?}",
-            src_ty,
-            dst_ty
-        ),
+        _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty),
     }
 }
 
 // Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs
 
 pub(crate) fn size_and_align_of_dst<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     info: Value,
 ) -> (Value, Value) {
     if !layout.is_unsized() {
-        let size = fx
-            .bcx
-            .ins()
-            .iconst(fx.pointer_type, layout.size.bytes() as i64);
-        let align = fx
-            .bcx
-            .ins()
-            .iconst(fx.pointer_type, layout.align.abi.bytes() as i64);
+        let size = fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64);
+        let align = fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64);
         return (size, align);
     }
     match layout.ty.kind() {
         ty::Dynamic(..) => {
             // load size/align from vtable
-            (
-                crate::vtable::size_of_obj(fx, info),
-                crate::vtable::min_align_of_obj(fx, info),
-            )
+            (crate::vtable::size_of_obj(fx, info), crate::vtable::min_align_of_obj(fx, info))
         }
         ty::Slice(_) | ty::Str => {
             let unit = layout.field(fx, 0);
@@ -171,9 +151,7 @@ pub(crate) fn size_and_align_of_dst<'tcx>(
             // times the unit size.
             (
                 fx.bcx.ins().imul_imm(info, unit.size.bytes() as i64),
-                fx.bcx
-                    .ins()
-                    .iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
+                fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
             )
         }
         _ => {
@@ -211,10 +189,7 @@ pub(crate) fn size_and_align_of_dst<'tcx>(
 
             // Choose max of two known alignments (combined value must
             // be aligned according to more restrictive of the two).
-            let cmp = fx
-                .bcx
-                .ins()
-                .icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align);
+            let cmp = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align);
             let align = fx.bcx.ins().select(cmp, sized_align, unsized_align);
 
             // Issue #27023: must add any necessary padding to `size`
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 765604e0f98..b97d3900984 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -2,11 +2,10 @@
 
 use crate::prelude::*;
 
-use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
 
 fn codegen_field<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     base: Pointer,
     extra: Option<Value>,
     layout: TyAndLayout<'tcx>,
@@ -15,11 +14,8 @@ fn codegen_field<'tcx>(
     let field_offset = layout.fields.offset(field.index());
     let field_layout = layout.field(&*fx, field.index());
 
-    let simple = |fx: &mut FunctionCx<'_, '_, _>| {
-        (
-            base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()),
-            field_layout,
-        )
+    let simple = |fx: &mut FunctionCx<'_, '_, '_>| {
+        (base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout)
     };
 
     if let Some(extra) = extra {
@@ -58,10 +54,7 @@ fn scalar_pair_calculate_b_offset(
     a_scalar: &Scalar,
     b_scalar: &Scalar,
 ) -> Offset32 {
-    let b_offset = a_scalar
-        .value
-        .size(&tcx)
-        .align_to(b_scalar.value.align(&tcx).abi);
+    let b_offset = a_scalar.value.size(&tcx).align_to(b_scalar.value.align(&tcx).abi);
     Offset32::new(b_offset.bytes().try_into().unwrap())
 }
 
@@ -106,10 +99,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     // FIXME remove
-    pub(crate) fn force_stack(
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    ) -> (Pointer, Option<Value>) {
+    pub(crate) fn force_stack(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Pointer, Option<Value>) {
         let layout = self.1;
         match self.0 {
             CValueInner::ByRef(ptr, meta) => (ptr, meta),
@@ -129,7 +119,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value with layout.abi of scalar
-    pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> Value {
+    pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value {
         let layout = self.1;
         match self.0 {
             CValueInner::ByRef(ptr, None) => {
@@ -153,10 +143,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value pair with layout.abi of scalar pair
-    pub(crate) fn load_scalar_pair(
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    ) -> (Value, Value) {
+    pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) {
         let layout = self.1;
         match self.0 {
             CValueInner::ByRef(ptr, None) => {
@@ -183,7 +170,7 @@ impl<'tcx> CValue<'tcx> {
 
     pub(crate) fn value_field(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         field: mir::Field,
     ) -> CValue<'tcx> {
         let layout = self.1;
@@ -219,21 +206,17 @@ impl<'tcx> CValue<'tcx> {
         }
     }
 
-    pub(crate) fn unsize_value(
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-        dest: CPlace<'tcx>,
-    ) {
+    pub(crate) fn unsize_value(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
         crate::unsize::coerce_unsized_into(fx, self, dest);
     }
 
     /// If `ty` is signed, `const_val` must already be sign extended.
     pub(crate) fn const_val(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
         const_val: ty::ScalarInt,
     ) -> CValue<'tcx> {
-        assert_eq!(const_val.size(), layout.size);
+        assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout);
         use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
 
         let clif_ty = fx.clif_type(layout.ty).unwrap();
@@ -250,18 +233,11 @@ impl<'tcx> CValue<'tcx> {
             ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
                 let const_val = const_val.to_bits(layout.size).unwrap();
                 let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64);
-                let msb = fx
-                    .bcx
-                    .ins()
-                    .iconst(types::I64, (const_val >> 64) as u64 as i64);
+                let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64);
                 fx.bcx.ins().iconcat(lsb, msb)
             }
-            ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..)
-            | ty::RawPtr(..) => {
-                fx
-                    .bcx
-                    .ins()
-                    .iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
+            ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) | ty::RawPtr(..) => {
+                fx.bcx.ins().iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
             }
             ty::Float(FloatTy::F32) => {
                 fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap()))
@@ -279,14 +255,8 @@ impl<'tcx> CValue<'tcx> {
     }
 
     pub(crate) fn cast_pointer_to(self, layout: TyAndLayout<'tcx>) -> Self {
-        assert!(matches!(
-            self.layout().ty.kind(),
-            ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)
-        ));
-        assert!(matches!(
-            layout.ty.kind(),
-            ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)
-        ));
+        assert!(matches!(self.layout().ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
+        assert!(matches!(layout.ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
         assert_eq!(self.layout().abi, layout.abi);
         CValue(self.0, layout)
     }
@@ -317,14 +287,11 @@ impl<'tcx> CPlace<'tcx> {
     }
 
     pub(crate) fn no_place(layout: TyAndLayout<'tcx>) -> CPlace<'tcx> {
-        CPlace {
-            inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None),
-            layout,
-        }
+        CPlace { inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None), layout }
     }
 
     pub(crate) fn new_stack_slot(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
         assert!(!layout.is_unsized());
@@ -339,28 +306,22 @@ impl<'tcx> CPlace<'tcx> {
             size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
             offset: None,
         });
-        CPlace {
-            inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None),
-            layout,
-        }
+        CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
     }
 
     pub(crate) fn new_var(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         local: Local,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
         let var = Variable::with_u32(fx.next_ssa_var);
         fx.next_ssa_var += 1;
         fx.bcx.declare_var(var, fx.clif_type(layout.ty).unwrap());
-        CPlace {
-            inner: CPlaceInner::Var(local, var),
-            layout,
-        }
+        CPlace { inner: CPlaceInner::Var(local, var), layout }
     }
 
     pub(crate) fn new_var_pair(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         local: Local,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
@@ -372,17 +333,11 @@ impl<'tcx> CPlace<'tcx> {
         let (ty1, ty2) = fx.clif_pair_type(layout.ty).unwrap();
         fx.bcx.declare_var(var1, ty1);
         fx.bcx.declare_var(var2, ty2);
-        CPlace {
-            inner: CPlaceInner::VarPair(local, var1, var2),
-            layout,
-        }
+        CPlace { inner: CPlaceInner::VarPair(local, var1, var2), layout }
     }
 
     pub(crate) fn for_ptr(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CPlace<'tcx> {
-        CPlace {
-            inner: CPlaceInner::Addr(ptr, None),
-            layout,
-        }
+        CPlace { inner: CPlaceInner::Addr(ptr, None), layout }
     }
 
     pub(crate) fn for_ptr_with_extra(
@@ -390,34 +345,27 @@ impl<'tcx> CPlace<'tcx> {
         extra: Value,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
-        CPlace {
-            inner: CPlaceInner::Addr(ptr, Some(extra)),
-            layout,
-        }
+        CPlace { inner: CPlaceInner::Addr(ptr, Some(extra)), layout }
     }
 
-    pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CValue<'tcx> {
+    pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CValue<'tcx> {
         let layout = self.layout();
         match self.inner {
             CPlaceInner::Var(_local, var) => {
                 let val = fx.bcx.use_var(var);
-                fx.bcx
-                    .set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
+                //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
                 CValue::by_val(val, layout)
             }
             CPlaceInner::VarPair(_local, var1, var2) => {
                 let val1 = fx.bcx.use_var(var1);
-                fx.bcx
-                    .set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index()));
+                //fx.bcx.set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index()));
                 let val2 = fx.bcx.use_var(var2);
-                fx.bcx
-                    .set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
+                //fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
                 CValue::by_val_pair(val1, val2, layout)
             }
             CPlaceInner::VarLane(_local, var, lane) => {
                 let val = fx.bcx.use_var(var);
-                fx.bcx
-                    .set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
+                //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
                 let val = fx.bcx.ins().extractlane(val, lane);
                 CValue::by_val(val, layout)
             }
@@ -447,11 +395,7 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
-    pub(crate) fn write_cvalue(
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-        from: CValue<'tcx>,
-    ) {
+    pub(crate) fn write_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>) {
         assert_assignable(fx, from.layout().ty, self.layout().ty);
 
         self.write_cvalue_maybe_transmute(fx, from, "write_cvalue");
@@ -459,7 +403,7 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn write_cvalue_transmute(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         from: CValue<'tcx>,
     ) {
         self.write_cvalue_maybe_transmute(fx, from, "write_cvalue_transmute");
@@ -467,12 +411,12 @@ impl<'tcx> CPlace<'tcx> {
 
     fn write_cvalue_maybe_transmute(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         from: CValue<'tcx>,
-        #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
+        method: &'static str,
     ) {
         fn transmute_value<'tcx>(
-            fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+            fx: &mut FunctionCx<'_, '_, 'tcx>,
             var: Variable,
             data: Value,
             dst_ty: Type,
@@ -511,15 +455,13 @@ impl<'tcx> CPlace<'tcx> {
                 }
                 _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty),
             };
-            fx.bcx
-                .set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
+            //fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
             fx.bcx.def_var(var, data);
         }
 
         assert_eq!(self.layout().size, from.layout().size);
 
-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
             use cranelift_codegen::cursor::{Cursor, CursorPosition};
             let cur_block = match fx.bcx.cursor().position() {
                 CursorPosition::After(block) => block,
@@ -558,15 +500,13 @@ impl<'tcx> CPlace<'tcx> {
 
                 // First get the old vector
                 let vector = fx.bcx.use_var(var);
-                fx.bcx
-                    .set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
+                //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
 
                 // Next insert the written lane into the vector
                 let vector = fx.bcx.ins().insertlane(vector, data, lane);
 
                 // Finally write the new vector
-                fx.bcx
-                    .set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
+                //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
                 fx.bcx.def_var(var, vector);
 
                 return;
@@ -604,10 +544,7 @@ impl<'tcx> CPlace<'tcx> {
                 to_ptr.store(fx, val, flags);
             }
             CValueInner::ByValPair(_, _) => {
-                bug!(
-                    "Non ScalarPair abi {:?} for ByValPair CValue",
-                    dst_layout.abi
-                );
+                bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
             }
             CValueInner::ByRef(from_ptr, None) => {
                 let from_addr = from_ptr.get_addr(fx);
@@ -632,7 +569,7 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn place_field(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         field: mir::Field,
     ) -> CPlace<'tcx> {
         let layout = self.layout();
@@ -650,18 +587,8 @@ impl<'tcx> CPlace<'tcx> {
                 let layout = layout.field(&*fx, field.index());
 
                 match field.as_u32() {
-                    0 => {
-                        return CPlace {
-                            inner: CPlaceInner::Var(local, var1),
-                            layout,
-                        }
-                    }
-                    1 => {
-                        return CPlace {
-                            inner: CPlaceInner::Var(local, var2),
-                            layout,
-                        }
-                    }
+                    0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout },
+                    1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout },
                     _ => unreachable!("field should be 0 or 1"),
                 }
             }
@@ -680,7 +607,7 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn place_index(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         index: Value,
     ) -> CPlace<'tcx> {
         let (elem_layout, ptr) = match self.layout().ty.kind() {
@@ -689,30 +616,24 @@ impl<'tcx> CPlace<'tcx> {
             _ => bug!("place_index({:?})", self.layout().ty),
         };
 
-        let offset = fx
-            .bcx
-            .ins()
-            .imul_imm(index, elem_layout.size.bytes() as i64);
+        let offset = fx.bcx.ins().imul_imm(index, elem_layout.size.bytes() as i64);
 
         CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout)
     }
 
-    pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CPlace<'tcx> {
+    pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> {
         let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty);
         if has_ptr_meta(fx.tcx, inner_layout.ty) {
             let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx);
             CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout)
         } else {
-            CPlace::for_ptr(
-                Pointer::new(self.to_cvalue(fx).load_scalar(fx)),
-                inner_layout,
-            )
+            CPlace::for_ptr(Pointer::new(self.to_cvalue(fx).load_scalar(fx)), inner_layout)
         }
     }
 
     pub(crate) fn place_ref(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
     ) -> CValue<'tcx> {
         if has_ptr_meta(fx.tcx, self.layout().ty) {
@@ -729,21 +650,18 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn downcast_variant(
         self,
-        fx: &FunctionCx<'_, 'tcx, impl Module>,
+        fx: &FunctionCx<'_, '_, 'tcx>,
         variant: VariantIdx,
     ) -> Self {
         assert!(!self.layout().is_unsized());
         let layout = self.layout().for_variant(fx, variant);
-        CPlace {
-            inner: self.inner,
-            layout,
-        }
+        CPlace { inner: self.inner, layout }
     }
 }
 
 #[track_caller]
 pub(crate) fn assert_assignable<'tcx>(
-    fx: &FunctionCx<'_, 'tcx, impl Module>,
+    fx: &FunctionCx<'_, '_, 'tcx>,
     from_ty: Ty<'tcx>,
     to_ty: Ty<'tcx>,
 ) {
@@ -776,12 +694,9 @@ pub(crate) fn assert_assignable<'tcx>(
         }
         (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => {
             for (from, to) in from_traits.iter().zip(to_traits) {
-                let from = fx
-                    .tcx
-                    .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
-                let to = fx
-                    .tcx
-                    .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
+                let from =
+                    fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
+                let to = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
                 assert_eq!(
                     from, to,
                     "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}",
@@ -790,6 +705,19 @@ pub(crate) fn assert_assignable<'tcx>(
             }
             // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
         }
+        (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b))
+            if adt_def_a.did == adt_def_b.did =>
+        {
+            let mut types_a = substs_a.types();
+            let mut types_b = substs_b.types();
+            loop {
+                match (types_a.next(), types_b.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b),
+                    (None, None) => return,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+        }
         _ => {
             assert_eq!(
                 from_ty, to_ty,
diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs
index 8f15586a9dc..9053d1aa1b0 100644
--- a/compiler/rustc_codegen_cranelift/src/vtable.rs
+++ b/compiler/rustc_codegen_cranelift/src/vtable.rs
@@ -1,6 +1,6 @@
 //! Codegen vtables and vtable accesses.
 //!
-//! See librustc_codegen_llvm/meth.rs for reference
+//! See `rustc_codegen_ssa/src/meth.rs` for reference.
 // FIXME dedup this logic between miri, cg_llvm and cg_clif
 
 use crate::prelude::*;
@@ -15,7 +15,7 @@ fn vtable_memflags() -> MemFlags {
     flags
 }
 
-pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
+pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
     let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
     fx.bcx.ins().load(
         pointer_ty(fx.tcx),
@@ -25,7 +25,7 @@ pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: V
     )
 }
 
-pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
+pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
     let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
     fx.bcx.ins().load(
         pointer_ty(fx.tcx),
@@ -35,7 +35,7 @@ pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Valu
     )
 }
 
-pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
+pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
     let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
     fx.bcx.ins().load(
         pointer_ty(fx.tcx),
@@ -46,7 +46,7 @@ pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable:
 }
 
 pub(crate) fn get_ptr_and_method_ref<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     arg: CValue<'tcx>,
     idx: usize,
 ) -> (Value, Value) {
@@ -68,7 +68,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
 }
 
 pub(crate) fn get_vtable<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
 ) -> Value {
@@ -85,7 +85,7 @@ pub(crate) fn get_vtable<'tcx>(
 }
 
 fn build_vtable<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
 ) -> DataId {
@@ -94,7 +94,7 @@ fn build_vtable<'tcx>(
 
     let drop_in_place_fn = import_function(
         tcx,
-        &mut fx.cx.module,
+        fx.cx.module,
         Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx),
     );
 
@@ -111,7 +111,7 @@ fn build_vtable<'tcx>(
         opt_mth.map(|(def_id, substs)| {
             import_function(
                 tcx,
-                &mut fx.cx.module,
+                fx.cx.module,
                 Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs)
                     .unwrap()
                     .polymorphize(fx.tcx),
@@ -165,11 +165,8 @@ fn build_vtable<'tcx>(
 }
 
 fn write_usize(tcx: TyCtxt<'_>, buf: &mut [u8], idx: usize, num: u64) {
-    let pointer_size = tcx
-        .layout_of(ParamEnv::reveal_all().and(tcx.types.usize))
-        .unwrap()
-        .size
-        .bytes() as usize;
+    let pointer_size =
+        tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.usize)).unwrap().size.bytes() as usize;
     let target = &mut buf[idx * pointer_size..(idx + 1) * pointer_size];
 
     match tcx.data_layout.endian {
diff --git a/compiler/rustc_codegen_cranelift/test.sh b/compiler/rustc_codegen_cranelift/test.sh
index 5ab10e0e905..e222adc7b80 100755
--- a/compiler/rustc_codegen_cranelift/test.sh
+++ b/compiler/rustc_codegen_cranelift/test.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 ./build.sh --sysroot none "$@"
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index d9393ffe534..854e3ccc21b 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -1,6 +1,7 @@
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 use crate::llvm::{self, AttributePlace};
+use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -41,12 +42,29 @@ impl ArgAttributeExt for ArgAttribute {
 }
 
 pub trait ArgAttributesExt {
-    fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value);
-    fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value);
+    fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value);
+    fn apply_attrs_to_callsite(
+        &self,
+        idx: AttributePlace,
+        cx: &CodegenCx<'_, '_>,
+        callsite: &Value,
+    );
+}
+
+fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool {
+    // LLVM prior to version 12 has known miscompiles in the presence of
+    // noalias attributes (see #54878). Only enable mutable noalias by
+    // default for versions we believe to be safe.
+    cx.tcx
+        .sess
+        .opts
+        .debugging_opts
+        .mutable_noalias
+        .unwrap_or_else(|| llvm_util::get_version() >= (12, 0, 0))
 }
 
 impl ArgAttributesExt for ArgAttributes {
-    fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value) {
+    fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) {
         let mut regular = self.regular;
         unsafe {
             let deref = self.pointee_size.bytes();
@@ -62,6 +80,9 @@ impl ArgAttributesExt for ArgAttributes {
                 llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32);
             }
             regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn));
+            if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
+                llvm::Attribute::NoAlias.apply_llfn(idx, llfn);
+            }
             match self.arg_ext {
                 ArgExtension::None => {}
                 ArgExtension::Zext => {
@@ -74,7 +95,12 @@ impl ArgAttributesExt for ArgAttributes {
         }
     }
 
-    fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value) {
+    fn apply_attrs_to_callsite(
+        &self,
+        idx: AttributePlace,
+        cx: &CodegenCx<'_, '_>,
+        callsite: &Value,
+    ) {
         let mut regular = self.regular;
         unsafe {
             let deref = self.pointee_size.bytes();
@@ -98,6 +124,9 @@ impl ArgAttributesExt for ArgAttributes {
                 );
             }
             regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite));
+            if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
+                llvm::Attribute::NoAlias.apply_callsite(idx, callsite);
+            }
             match self.arg_ext {
                 ArgExtension::None => {}
                 ArgExtension::Zext => {
@@ -419,13 +448,13 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
 
         let mut i = 0;
         let mut apply = |attrs: &ArgAttributes| {
-            attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), llfn);
+            attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn);
             i += 1;
             i - 1
         };
         match self.ret.mode {
             PassMode::Direct(ref attrs) => {
-                attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, llfn);
+                attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
             }
             PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
                 assert!(!on_stack);
@@ -480,18 +509,18 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         // FIXME(wesleywiser, eddyb): We should apply `nounwind` and `noreturn` as appropriate to this callsite.
 
         let mut i = 0;
-        let mut apply = |attrs: &ArgAttributes| {
-            attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), callsite);
+        let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| {
+            attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite);
             i += 1;
             i - 1
         };
         match self.ret.mode {
             PassMode::Direct(ref attrs) => {
-                attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, callsite);
+                attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, &bx.cx, callsite);
             }
             PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
                 assert!(!on_stack);
-                let i = apply(attrs);
+                let i = apply(bx.cx, attrs);
                 unsafe {
                     llvm::LLVMRustAddStructRetCallSiteAttr(
                         callsite,
@@ -517,12 +546,12 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         }
         for arg in &self.args {
             if arg.pad.is_some() {
-                apply(&ArgAttributes::new());
+                apply(bx.cx, &ArgAttributes::new());
             }
             match arg.mode {
                 PassMode::Ignore => {}
                 PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
-                    let i = apply(attrs);
+                    let i = apply(bx.cx, attrs);
                     unsafe {
                         llvm::LLVMRustAddByValCallSiteAttr(
                             callsite,
@@ -533,22 +562,22 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 }
                 PassMode::Direct(ref attrs)
                 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
-                    apply(attrs);
+                    apply(bx.cx, attrs);
                 }
                 PassMode::Indirect {
                     ref attrs,
                     extra_attrs: Some(ref extra_attrs),
                     on_stack: _,
                 } => {
-                    apply(attrs);
-                    apply(extra_attrs);
+                    apply(bx.cx, attrs);
+                    apply(bx.cx, extra_attrs);
                 }
                 PassMode::Pair(ref a, ref b) => {
-                    apply(a);
-                    apply(b);
+                    apply(bx.cx, a);
+                    apply(bx.cx, b);
                 }
                 PassMode::Cast(_) => {
-                    apply(&ArgAttributes::new());
+                    apply(bx.cx, &ArgAttributes::new());
                 }
             }
         }
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index 38c8ae711a4..84b091d8d4d 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -14,7 +14,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_hir as hir;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::{bug, span_bug};
-use rustc_span::{Pos, Span};
+use rustc_span::{Pos, Span, Symbol};
 use rustc_target::abi::*;
 use rustc_target::asm::*;
 
@@ -125,15 +125,39 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 
         // Collect the types of output operands
         let mut constraints = vec![];
+        let mut clobbers = vec![];
         let mut output_types = vec![];
         let mut op_idx = FxHashMap::default();
         for (idx, op) in operands.iter().enumerate() {
             match *op {
                 InlineAsmOperandRef::Out { reg, late, place } => {
+                    let is_target_supported = |reg_class: InlineAsmRegClass| {
+                        for &(_, feature) in reg_class.supported_types(asm_arch) {
+                            if let Some(feature) = feature {
+                                if self.tcx.sess.target_features.contains(&Symbol::intern(feature))
+                                {
+                                    return true;
+                                }
+                            } else {
+                                // Register class is unconditionally supported
+                                return true;
+                            }
+                        }
+                        false
+                    };
+
                     let mut layout = None;
                     let ty = if let Some(ref place) = place {
                         layout = Some(&place.layout);
                         llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
+                    } else if !is_target_supported(reg.reg_class()) {
+                        // We turn discarded outputs into clobber constraints
+                        // if the target feature needed by the register class is
+                        // disabled. This is necessary otherwise LLVM will try
+                        // to actually allocate a register for the dummy output.
+                        assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_)));
+                        clobbers.push(format!("~{}", reg_to_llvm(reg, None)));
+                        continue;
                     } else {
                         // If the output is discarded, we don't really care what
                         // type is used. We're just using this to tell LLVM to
@@ -244,6 +268,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             }
         }
 
+        constraints.append(&mut clobbers);
         if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
             match asm_arch {
                 InlineAsmArch::AArch64 | InlineAsmArch::Arm => {
@@ -528,6 +553,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>)
             InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
                 bug!("LLVM backend does not support SPIR-V")
             }
+            InlineAsmRegClass::Err => unreachable!(),
         }
         .to_string(),
     }
@@ -594,6 +620,7 @@ fn modifier_to_llvm(
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
         }
+        InlineAsmRegClass::Err => unreachable!(),
     }
 }
 
@@ -637,6 +664,7 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
         }
+        InlineAsmRegClass::Err => unreachable!(),
     }
 }
 
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index 09ece6164eb..9e5e2b1039e 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -11,9 +11,10 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_session::config::{OptLevel, SanitizerSet};
+use rustc_session::config::OptLevel;
 use rustc_session::Session;
-use rustc_target::spec::StackProbeType;
+use rustc_target::spec::abi::Abi;
+use rustc_target::spec::{SanitizerSet, StackProbeType};
 
 use crate::attributes;
 use crate::llvm::AttributePlace::Function;
@@ -152,18 +153,6 @@ fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
     }
 }
 
-pub fn llvm_target_features(sess: &Session) -> impl Iterator<Item = &str> {
-    const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"];
-
-    let cmdline = sess
-        .opts
-        .cg
-        .target_feature
-        .split(',')
-        .filter(|f| !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s)));
-    sess.target.features.split(',').chain(cmdline).filter(|l| !l.is_empty())
-}
-
 pub fn apply_target_cpu_attr(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
     let target_cpu = SmallCStr::new(llvm_util::target_cpu(cx.tcx.sess));
     llvm::AddFunctionAttrStringValue(
@@ -266,6 +255,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
         attributes::emit_uwtable(llfn, true);
     }
 
+    // FIXME: none of these three functions interact with source level attributes.
     set_frame_pointer_elimination(cx, llfn);
     set_instrument_function(cx, llfn);
     set_probestack(cx, llfn);
@@ -291,6 +281,9 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
     if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) {
         llvm::AddFunctionAttrString(llfn, Function, cstr!("cmse_nonsecure_entry"));
     }
+    if let Some(align) = codegen_fn_attrs.alignment {
+        llvm::set_alignment(llfn, align as usize);
+    }
     sanitize(cx, codegen_fn_attrs.no_sanitize, llfn);
 
     // Always annotate functions with the target-cpu they are compiled for.
@@ -301,33 +294,22 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
     // The target doesn't care; the subtarget reads our attribute.
     apply_tune_cpu_attr(cx, llfn);
 
-    let features = llvm_target_features(cx.tcx.sess)
-        .map(|s| s.to_string())
-        .chain(codegen_fn_attrs.target_features.iter().map(|f| {
+    let mut function_features = codegen_fn_attrs
+        .target_features
+        .iter()
+        .map(|f| {
             let feature = &f.as_str();
             format!("+{}", llvm_util::to_llvm_feature(cx.tcx.sess, feature))
-        }))
+        })
         .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x {
             InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(),
             InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(),
         }))
-        .collect::<Vec<String>>()
-        .join(",");
+        .collect::<Vec<String>>();
 
-    if !features.is_empty() {
-        let val = CString::new(features).unwrap();
-        llvm::AddFunctionAttrStringValue(
-            llfn,
-            llvm::AttributePlace::Function,
-            cstr!("target-features"),
-            &val,
-        );
-    }
-
-    // Note that currently the `wasm-import-module` doesn't do anything, but
-    // eventually LLVM 7 should read this and ferry the appropriate import
-    // module to the output file.
-    if cx.tcx.sess.target.arch == "wasm32" {
+    if cx.tcx.sess.target.is_like_wasm {
+        // If this function is an import from the environment but the wasm
+        // import has a specific module/name, apply them here.
         if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) {
             llvm::AddFunctionAttrStringValue(
                 llfn,
@@ -346,6 +328,30 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
                 &name,
             );
         }
+
+        // The `"wasm"` abi on wasm targets automatically enables the
+        // `+multivalue` feature because the purpose of the wasm abi is to match
+        // the WebAssembly specification, which has this feature. This won't be
+        // needed when LLVM enables this `multivalue` feature by default.
+        if !cx.tcx.is_closure(instance.def_id()) {
+            let abi = cx.tcx.fn_sig(instance.def_id()).abi();
+            if abi == Abi::Wasm {
+                function_features.push("+multivalue".to_string());
+            }
+        }
+    }
+
+    if !function_features.is_empty() {
+        let mut global_features = llvm_util::llvm_global_features(cx.tcx.sess);
+        global_features.extend(function_features.into_iter());
+        let features = global_features.join(",");
+        let val = CString::new(features).unwrap();
+        llvm::AddFunctionAttrStringValue(
+            llfn,
+            llvm::AttributePlace::Function,
+            cstr!("target-features"),
+            &val,
+        );
     }
 }
 
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index 5effe687528..4226ed7d99b 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -24,6 +24,7 @@ use tracing::{debug, info};
 use std::ffi::{CStr, CString};
 use std::fs::File;
 use std::io;
+use std::iter;
 use std::path::Path;
 use std::ptr;
 use std::slice;
@@ -916,9 +917,7 @@ impl ThinLTOKeysMap {
         modules: &[llvm::ThinLTOModule],
         names: &[CString],
     ) -> Self {
-        let keys = modules
-            .iter()
-            .zip(names.iter())
+        let keys = iter::zip(modules, names)
             .map(|(module, name)| {
                 let key = build_string(|rust_str| unsafe {
                     llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0);
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index c224da7885b..b628ae3ae3a 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -1,4 +1,3 @@
-use crate::attributes;
 use crate::back::lto::ThinBuffer;
 use crate::back::profiling::{
     selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler,
@@ -24,11 +23,11 @@ use rustc_fs_util::{link_or_copy, path_to_c_string};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{self, Lto, OutputType, Passes, SanitizerSet, SwitchWithOptPath};
+use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
-use rustc_target::spec::{CodeModel, RelocModel, SplitDebuginfo};
+use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo};
 use tracing::debug;
 
 use libc::{c_char, c_int, c_uint, c_void, size_t};
@@ -140,7 +139,7 @@ fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
     }
 }
 
-fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
+pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
     match code_model {
         Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
         Some(CodeModel::Small) => llvm::CodeModel::Small,
@@ -166,23 +165,18 @@ pub fn target_machine_factory(
 
     let code_model = to_llvm_code_model(sess.code_model());
 
-    let mut features = llvm_util::handle_native_features(sess);
-    features.extend(attributes::llvm_target_features(sess).map(|s| s.to_owned()));
     let mut singlethread = sess.target.singlethread;
 
     // On the wasm target once the `atomics` feature is enabled that means that
     // we're no longer single-threaded, or otherwise we don't want LLVM to
     // lower atomic operations to single-threaded operations.
-    if singlethread
-        && sess.target.llvm_target.contains("wasm32")
-        && sess.target_features.contains(&sym::atomics)
-    {
+    if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
         singlethread = false;
     }
 
     let triple = SmallCStr::new(&sess.target.llvm_target);
     let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
-    let features = features.join(",");
+    let features = llvm_util::llvm_global_features(sess).join(",");
     let features = CString::new(features).unwrap();
     let abi = SmallCStr::new(&sess.target.llvm_abiname);
     let trap_unreachable =
@@ -551,6 +545,15 @@ pub(crate) unsafe fn optimize(
                     llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap());
                     continue;
                 }
+                if pass_name == "insert-gcov-profiling" || pass_name == "instrprof" {
+                    // Instrumentation must be inserted before optimization,
+                    // otherwise LLVM may optimize some functions away which
+                    // breaks llvm-cov.
+                    //
+                    // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp.
+                    llvm::LLVMRustAddPass(mpm, find_pass(pass_name).unwrap());
+                    continue;
+                }
 
                 if let Some(pass) = find_pass(pass_name) {
                     extra_passes.push(pass);
@@ -1044,7 +1047,7 @@ pub unsafe fn with_llvm_pmb(
     // thresholds copied from clang.
     match (opt_level, opt_size, inline_threshold) {
         (.., Some(t)) => {
-            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t as u32);
+            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t);
         }
         (llvm::CodeGenOptLevel::Aggressive, ..) => {
             llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275);
diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs
index d5be3132dee..6f6c649bb0b 100644
--- a/compiler/rustc_codegen_llvm/src/base.rs
+++ b/compiler/rustc_codegen_llvm/src/base.rs
@@ -32,8 +32,9 @@ use rustc_middle::middle::cstore::EncodedMetadata;
 use rustc_middle::middle::exported_symbols;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{DebugInfo, SanitizerSet};
+use rustc_session::config::DebugInfo;
 use rustc_span::symbol::Symbol;
+use rustc_target::spec::SanitizerSet;
 
 use std::ffi::CString;
 use std::time::Instant;
@@ -143,7 +144,7 @@ pub fn compile_codegen_unit(
 
             // Finalize code coverage by injecting the coverage map. Note, the coverage map will
             // also be added to the `llvm.used` variable, created next.
-            if cx.sess().opts.debugging_opts.instrument_coverage {
+            if cx.sess().instrument_coverage() {
                 cx.coverageinfo_finalize();
             }
 
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index f4852c91e53..053cda1e7cc 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -2,6 +2,7 @@ use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm::{self, BasicBlock, False};
 use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope};
+use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -16,11 +17,12 @@ use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::{sym, Span};
+use rustc_span::Span;
 use rustc_target::abi::{self, Align, Size};
 use rustc_target::spec::{HasTargetSpec, Target};
 use std::borrow::Cow;
 use std::ffi::CStr;
+use std::iter;
 use std::ops::{Deref, Range};
 use std::ptr;
 use tracing::debug;
@@ -260,7 +262,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -268,7 +270,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -276,7 +278,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -284,7 +286,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -292,7 +294,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -668,81 +670,47 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
-        // WebAssembly has saturating floating point to integer casts if the
-        // `nontrapping-fptoint` target feature is activated. We'll use those if
-        // they are available.
-        if self.sess().target.arch == "wasm32"
-            && self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
-        {
+        if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() {
             let src_ty = self.cx.val_ty(val);
             let float_width = self.cx.float_width(src_ty);
             let int_width = self.cx.int_width(dest_ty);
-            let name = match (int_width, float_width) {
-                (32, 32) => Some("llvm.wasm.trunc.saturate.unsigned.i32.f32"),
-                (32, 64) => Some("llvm.wasm.trunc.saturate.unsigned.i32.f64"),
-                (64, 32) => Some("llvm.wasm.trunc.saturate.unsigned.i64.f32"),
-                (64, 64) => Some("llvm.wasm.trunc.saturate.unsigned.i64.f64"),
-                _ => None,
-            };
-            if let Some(name) = name {
-                let intrinsic = self.get_intrinsic(name);
-                return Some(self.call(intrinsic, &[val], None));
-            }
+            let name = format!("llvm.fptoui.sat.i{}.f{}", int_width, float_width);
+            let intrinsic = self.get_intrinsic(&name);
+            return Some(self.call(intrinsic, &[val], None));
         }
+
         None
     }
 
     fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
-        // WebAssembly has saturating floating point to integer casts if the
-        // `nontrapping-fptoint` target feature is activated. We'll use those if
-        // they are available.
-        if self.sess().target.arch == "wasm32"
-            && self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
-        {
+        if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() {
             let src_ty = self.cx.val_ty(val);
             let float_width = self.cx.float_width(src_ty);
             let int_width = self.cx.int_width(dest_ty);
-            let name = match (int_width, float_width) {
-                (32, 32) => Some("llvm.wasm.trunc.saturate.signed.i32.f32"),
-                (32, 64) => Some("llvm.wasm.trunc.saturate.signed.i32.f64"),
-                (64, 32) => Some("llvm.wasm.trunc.saturate.signed.i64.f32"),
-                (64, 64) => Some("llvm.wasm.trunc.saturate.signed.i64.f64"),
-                _ => None,
-            };
-            if let Some(name) = name {
-                let intrinsic = self.get_intrinsic(name);
-                return Some(self.call(intrinsic, &[val], None));
-            }
+            let name = format!("llvm.fptosi.sat.i{}.f{}", int_width, float_width);
+            let intrinsic = self.get_intrinsic(&name);
+            return Some(self.call(intrinsic, &[val], None));
         }
-        None
-    }
 
-    fn fptosui_may_trap(&self, val: &'ll Value, dest_ty: &'ll Type) -> bool {
-        // Most of the time we'll be generating the `fptosi` or `fptoui`
-        // instruction for floating-point-to-integer conversions. These
-        // instructions by definition in LLVM do not trap. For the WebAssembly
-        // target, however, we'll lower in some cases to intrinsic calls instead
-        // which may trap. If we detect that this is a situation where we'll be
-        // using the intrinsics then we report that the call map trap, which
-        // callers might need to handle.
-        if !self.wasm_and_missing_nontrapping_fptoint() {
-            return false;
-        }
-        let src_ty = self.cx.val_ty(val);
-        let float_width = self.cx.float_width(src_ty);
-        let int_width = self.cx.int_width(dest_ty);
-        matches!((int_width, float_width), (32, 32) | (32, 64) | (64, 32) | (64, 64))
+        None
     }
 
     fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
-        // When we can, use the native wasm intrinsics which have tighter
-        // codegen. Note that this has a semantic difference in that the
-        // intrinsic can trap whereas `fptoui` never traps. That difference,
-        // however, is handled by `fptosui_may_trap` above.
+        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
+        // poor codegen. The reason for this is that the corresponding wasm
+        // instructions, `i32.trunc_f32_s` for example, will trap when the float
+        // is out-of-bounds, infinity, or nan. This means that LLVM
+        // automatically inserts control flow around `fptoui` and `fptosi`
+        // because the LLVM instruction `fptoui` is defined as producing a
+        // poison value, not having UB on out-of-bounds values.
         //
-        // Note that we skip the wasm intrinsics for vector types where `fptoui`
-        // must be used instead.
-        if self.wasm_and_missing_nontrapping_fptoint() {
+        // This method, however, is only used with non-saturating casts that
+        // have UB on out-of-bounds values. This means that it's ok if we use
+        // the raw wasm instruction since out-of-bounds values can do whatever
+        // we like. To ensure that LLVM picks the right instruction we choose
+        // the raw wasm intrinsic functions which avoid LLVM inserting all the
+        // other control flow automatically.
+        if self.sess().target.arch == "wasm32" {
             let src_ty = self.cx.val_ty(val);
             if self.cx.type_kind(src_ty) != TypeKind::Vector {
                 let float_width = self.cx.float_width(src_ty);
@@ -764,7 +732,8 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
-        if self.wasm_and_missing_nontrapping_fptoint() {
+        // see `fptoui` above for why wasm is different here
+        if self.sess().target.arch == "wasm32" {
             let src_ty = self.cx.val_ty(val);
             if self.cx.type_kind(src_ty) != TypeKind::Vector {
                 let float_width = self.cx.float_width(src_ty);
@@ -1241,14 +1210,14 @@ impl Builder<'a, 'll, 'tcx> {
     pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
     pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -1281,7 +1250,7 @@ impl Builder<'a, 'll, 'tcx> {
         unsafe {
             let instr =
                 llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -1289,7 +1258,7 @@ impl Builder<'a, 'll, 'tcx> {
         unsafe {
             let instr =
                 llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -1352,18 +1321,14 @@ impl Builder<'a, 'll, 'tcx> {
 
         let param_tys = self.cx.func_params_types(fn_ty);
 
-        let all_args_match = param_tys
-            .iter()
-            .zip(args.iter().map(|&v| self.val_ty(v)))
+        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.val_ty(v)))
             .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
 
         if all_args_match {
             return Cow::Borrowed(args);
         }
 
-        let casted_args: Vec<_> = param_tys
-            .into_iter()
-            .zip(args.iter())
+        let casted_args: Vec<_> = iter::zip(param_tys, args)
             .enumerate()
             .map(|(i, (expected_ty, &actual_val))| {
                 let actual_ty = self.val_ty(actual_val);
@@ -1423,8 +1388,11 @@ impl Builder<'a, 'll, 'tcx> {
         }
     }
 
-    fn wasm_and_missing_nontrapping_fptoint(&self) -> bool {
-        self.sess().target.arch == "wasm32"
-            && !self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
+    fn fptoint_sat_broken_in_llvm(&self) -> bool {
+        match self.tcx.sess.target.arch.as_str() {
+            // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083
+            "riscv64" => llvm_util::get_version() < (13, 0, 0),
+            _ => false,
+        }
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs
index 367c1f4811c..b26969a5012 100644
--- a/compiler/rustc_codegen_llvm/src/callee.rs
+++ b/compiler/rustc_codegen_llvm/src/callee.rs
@@ -14,6 +14,7 @@ use tracing::debug;
 
 use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
 use rustc_middle::ty::{self, Instance, TypeFoldable};
+use rustc_target::spec::RelocModel;
 
 /// Codegens a reference to a fn/method item, monomorphizing and
 /// inlining as it goes.
@@ -170,17 +171,19 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value
                     }
                 }
             }
-        }
 
-        // MinGW: For backward compatibility we rely on the linker to decide whether it
-        // should use dllimport for functions.
-        if cx.use_dll_storage_attrs
-            && tcx.is_dllimport_foreign_item(instance_def_id)
-            && tcx.sess.target.env != "gnu"
-        {
-            unsafe {
+            // MinGW: For backward compatibility we rely on the linker to decide whether it
+            // should use dllimport for functions.
+            if cx.use_dll_storage_attrs
+                && tcx.is_dllimport_foreign_item(instance_def_id)
+                && tcx.sess.target.env != "gnu"
+            {
                 llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport);
             }
+
+            if cx.tcx.sess.relocation_model() == RelocModel::Static {
+                llvm::LLVMRustSetDSOLocal(llfn, true);
+            }
         }
 
         llfn
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 9f6a2ae3ca1..f5c54b11c08 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -1,4 +1,5 @@
 use crate::attributes;
+use crate::back::write::to_llvm_code_model;
 use crate::callee::get_fn;
 use crate::coverageinfo;
 use crate::debuginfo;
@@ -78,7 +79,7 @@ pub struct CodegenCx<'ll, 'tcx> {
     pub pointee_infos: RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>>,
     pub isize_ty: &'ll Type,
 
-    pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'tcx>>,
+    pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>,
     pub dbg_cx: Option<debuginfo::CrateDebugContext<'ll, 'tcx>>,
 
     eh_personality: Cell<Option<&'ll Value>>,
@@ -100,10 +101,6 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode {
     }
 }
 
-fn strip_x86_address_spaces(data_layout: String) -> String {
-    data_layout.replace("-p270:32:32-p271:32:32-p272:64:64-", "-")
-}
-
 fn strip_powerpc64_vectors(data_layout: String) -> String {
     data_layout.replace("-v256:256:256-v512:512:512", "")
 }
@@ -118,11 +115,6 @@ pub unsafe fn create_module(
     let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx);
 
     let mut target_data_layout = sess.target.data_layout.clone();
-    if llvm_util::get_version() < (10, 0, 0)
-        && (sess.target.arch == "x86" || sess.target.arch == "x86_64")
-    {
-        target_data_layout = strip_x86_address_spaces(target_data_layout);
-    }
     if llvm_util::get_version() < (12, 0, 0) && sess.target.arch == "powerpc64" {
         target_data_layout = strip_powerpc64_vectors(target_data_layout);
     }
@@ -181,6 +173,13 @@ pub unsafe fn create_module(
         }
     }
 
+    // Linking object files with different code models is undefined behavior
+    // because the compiler would have to generate additional code (to span
+    // longer jumps) if a larger code model is used with a smaller one.
+    //
+    // See https://reviews.llvm.org/D52322 and https://reviews.llvm.org/D52323.
+    llvm::LLVMRustSetModuleCodeModel(llmod, to_llvm_code_model(sess.code_model()));
+
     // If skipping the PLT is enabled, we need to add some module metadata
     // to ensure intrinsic calls don't use it.
     if !sess.needs_plt() {
@@ -272,7 +271,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
 
         let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod());
 
-        let coverage_cx = if tcx.sess.opts.debugging_opts.instrument_coverage {
+        let coverage_cx = if tcx.sess.instrument_coverage() {
             let covctx = coverageinfo::CrateCoverageContext::new();
             Some(covctx)
         } else {
@@ -323,7 +322,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     }
 
     #[inline]
-    pub fn coverage_context(&'a self) -> Option<&'a coverageinfo::CrateCoverageContext<'tcx>> {
+    pub fn coverage_context(&'a self) -> Option<&'a coverageinfo::CrateCoverageContext<'ll, 'tcx>> {
         self.coverage_cx.as_ref()
     }
 }
@@ -504,14 +503,6 @@ impl CodegenCx<'b, 'tcx> {
         let t_f32 = self.type_f32();
         let t_f64 = self.type_f64();
 
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i64.f64", fn(t_f64) -> t_i64);
-        ifn!("llvm.wasm.trunc.saturate.signed.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.signed.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.signed.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.wasm.trunc.saturate.signed.i64.f64", fn(t_f64) -> t_i64);
         ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32);
         ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32);
         ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64);
@@ -521,6 +512,28 @@ impl CodegenCx<'b, 'tcx> {
         ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64);
         ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64);
 
+        ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8);
+        ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16);
+        ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32);
+        ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64);
+        ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128);
+        ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8);
+        ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16);
+        ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32);
+        ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64);
+        ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128);
+
+        ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8);
+        ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16);
+        ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32);
+        ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64);
+        ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128);
+        ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8);
+        ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16);
+        ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32);
+        ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64);
+        ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128);
+
         ifn!("llvm.trap", fn() -> void);
         ifn!("llvm.debugtrap", fn() -> void);
         ifn!("llvm.frameaddress", fn(t_i32) -> i8p);
@@ -704,7 +717,7 @@ impl CodegenCx<'b, 'tcx> {
         ifn!("llvm.va_end", fn(i8p) -> void);
         ifn!("llvm.va_copy", fn(i8p, i8p) -> void);
 
-        if self.sess().opts.debugging_opts.instrument_coverage {
+        if self.sess().instrument_coverage() {
             ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void);
         }
 
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index 444a9d4ba04..2ac814bf228 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -3,13 +3,12 @@ use crate::coverageinfo;
 use crate::llvm;
 
 use llvm::coverageinfo::CounterMappingRegion;
-use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression, FunctionCoverage};
-use rustc_codegen_ssa::traits::ConstMethods;
+use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression};
+use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE};
 use rustc_llvm::RustString;
 use rustc_middle::mir::coverage::CodeRegion;
-use rustc_middle::ty::{Instance, TyCtxt};
 use rustc_span::Symbol;
 
 use std::ffi::CString;
@@ -20,16 +19,17 @@ use tracing::debug;
 ///
 /// This Coverage Map complies with Coverage Mapping Format version 4 (zero-based encoded as 3),
 /// as defined at [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format)
-/// and published in Rust's current (November 2020) fork of LLVM. This version is supported by the
-/// LLVM coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM.
+/// and published in Rust's November 2020 fork of LLVM. This version is supported by the LLVM
+/// coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM.
 ///
 /// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with
-/// version 3. Clang's implementation of Coverage Map generation was referenced when implementing
-/// this Rust version, and though the format documentation is very explicit and detailed, some
-/// undocumented details in Clang's implementation (that may or may not be important) were also
-/// replicated for Rust's Coverage Map.
+/// the same version. Clang's implementation of Coverage Map generation was referenced when
+/// implementing this Rust version, and though the format documentation is very explicit and
+/// detailed, some undocumented details in Clang's implementation (that may or may not be important)
+/// were also replicated for Rust's Coverage Map.
 pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     let tcx = cx.tcx;
+
     // Ensure LLVM supports Coverage Map Version 4 (encoded as a zero-based value: 3).
     // If not, the LLVM Version must be less than 11.
     let version = coverageinfo::mapping_version();
@@ -39,17 +39,24 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
 
     debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name());
 
-    let mut function_coverage_map = match cx.coverage_context() {
+    // In order to show that unused functions have coverage counts of zero (0), LLVM requires the
+    // functions exist. Generate synthetic functions with a (required) single counter, and add the
+    // MIR `Coverage` code regions to the `function_coverage_map`, before calling
+    // `ctx.take_function_coverage_map()`.
+    if !tcx.sess.instrument_coverage_except_unused_functions() {
+        add_unused_functions(cx);
+    }
+
+    let function_coverage_map = match cx.coverage_context() {
         Some(ctx) => ctx.take_function_coverage_map(),
         None => return,
     };
+
     if function_coverage_map.is_empty() {
         // This module has no functions with coverage instrumentation
         return;
     }
 
-    add_unreachable_coverage(tcx, &mut function_coverage_map);
-
     let mut mapgen = CoverageMapGenerator::new();
 
     // Encode coverage mappings and generate function records
@@ -57,7 +64,8 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     for (instance, function_coverage) in function_coverage_map {
         debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance);
         let mangled_function_name = tcx.symbol_name(instance).to_string();
-        let function_source_hash = function_coverage.source_hash();
+        let source_hash = function_coverage.source_hash();
+        let is_used = function_coverage.is_used();
         let (expressions, counter_regions) =
             function_coverage.get_expressions_and_counter_regions();
 
@@ -69,7 +77,7 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
             "Every `FunctionCoverage` should have at least one counter"
         );
 
-        function_data.push((mangled_function_name, function_source_hash, coverage_mapping_buffer));
+        function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer));
     }
 
     // Encode all filenames referenced by counters/expressions in this module
@@ -84,13 +92,14 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     // Generate the LLVM IR representation of the coverage map and store it in a well-known global
     let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val);
 
-    for (mangled_function_name, function_source_hash, coverage_mapping_buffer) in function_data {
+    for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data {
         save_function_record(
             cx,
             mangled_function_name,
-            function_source_hash,
+            source_hash,
             filenames_ref,
             coverage_mapping_buffer,
+            is_used,
         );
     }
 
@@ -201,9 +210,10 @@ impl CoverageMapGenerator {
 fn save_function_record(
     cx: &CodegenCx<'ll, 'tcx>,
     mangled_function_name: String,
-    function_source_hash: u64,
+    source_hash: u64,
     filenames_ref: u64,
     coverage_mapping_buffer: Vec<u8>,
+    is_used: bool,
 ) {
     // Concatenate the encoded coverage mappings
     let coverage_mapping_size = coverage_mapping_buffer.len();
@@ -212,128 +222,120 @@ fn save_function_record(
     let func_name_hash = coverageinfo::hash_str(&mangled_function_name);
     let func_name_hash_val = cx.const_u64(func_name_hash);
     let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32);
-    let func_hash_val = cx.const_u64(function_source_hash);
+    let source_hash_val = cx.const_u64(source_hash);
     let filenames_ref_val = cx.const_u64(filenames_ref);
     let func_record_val = cx.const_struct(
         &[
             func_name_hash_val,
             coverage_mapping_size_val,
-            func_hash_val,
+            source_hash_val,
             filenames_ref_val,
             coverage_mapping_val,
         ],
         /*packed=*/ true,
     );
 
-    // At the present time, the coverage map for Rust assumes every instrumented function `is_used`.
-    // Note that Clang marks functions as "unused" in `CodeGenPGO::emitEmptyCounterMapping`. (See:
-    // https://github.com/rust-lang/llvm-project/blob/de02a75e398415bad4df27b4547c25b896c8bf3b/clang%2Flib%2FCodeGen%2FCodeGenPGO.cpp#L877-L878
-    // for example.)
-    //
-    // It's not yet clear if or how this may be applied to Rust in the future, but the `is_used`
-    // argument is available and handled similarly.
-    let is_used = true;
     coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used);
 }
 
 /// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for
 /// the functions that went through codegen; such as public functions and "used" functions
 /// (functions referenced by other "used" or public items). Any other functions considered unused,
-/// or "Unreachable" were still parsed and processed through the MIR stage.
+/// or "Unreachable", were still parsed and processed through the MIR stage, but were not
+/// codegenned. (Note that `-Clink-dead-code` can force some unused code to be codegenned, but
+/// that flag is known to cause other errors, when combined with `-Z instrument-coverage`; and
+/// `-Clink-dead-code` will not generate code for unused generic functions.)
 ///
-/// We can find the unreachable functions by the set difference of all MIR `DefId`s (`tcx` query
-/// `mir_keys`) minus the codegenned `DefId`s (`tcx` query `collect_and_partition_mono_items`).
+/// We can find the unused functions (including generic functions) by the set difference of all MIR
+/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query
+/// `collect_and_partition_mono_items`).
 ///
 /// *HOWEVER* the codegenned `DefId`s are partitioned across multiple `CodegenUnit`s (CGUs), and
 /// this function is processing a `function_coverage_map` for the functions (`Instance`/`DefId`)
-/// allocated to only one of those CGUs. We must NOT inject any "Unreachable" functions's
-/// `CodeRegion`s more than once, so we have to pick which CGU's `function_coverage_map` to add
-/// each "Unreachable" function to.
-///
-/// Some constraints:
-///
-/// 1. The file name of an "Unreachable" function must match the file name of the existing
-///    codegenned (covered) function to which the unreachable code regions will be added.
-/// 2. The function to which the unreachable code regions will be added must not be a genaric
-///    function (must not have type parameters) because the coverage tools will get confused
-///    if the codegenned function has more than one instantiation and additional `CodeRegion`s
-///    attached to only one of those instantiations.
-fn add_unreachable_coverage<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    function_coverage_map: &mut FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>,
-) {
+/// allocated to only one of those CGUs. We must NOT inject any unused functions's `CodeRegion`s
+/// more than once, so we have to pick a CGUs `function_coverage_map` into which the unused
+/// function will be inserted.
+fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
+    let tcx = cx.tcx;
+
     // FIXME(#79622): Can this solution be simplified and/or improved? Are there other sources
     // of compiler state data that might help (or better sources that could be exposed, but
     // aren't yet)?
 
-    // Note: If the crate *only* defines generic functions, there are no codegenerated non-generic
-    // functions to add any unreachable code to. In this case, the unreachable code regions will
-    // have no coverage, instead of having coverage with zero executions.
-    //
-    // This is probably still an improvement over Clang, which does not generate any coverage
-    // for uninstantiated template functions.
-
-    let has_non_generic_def_ids =
-        function_coverage_map.keys().any(|instance| instance.def.attrs(tcx).len() == 0);
-
-    if !has_non_generic_def_ids {
-        // There are no non-generic functions to add unreachable `CodeRegion`s to
-        return;
-    }
+    let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics();
 
-    let all_def_ids: DefIdSet =
-        tcx.mir_keys(LOCAL_CRATE).iter().map(|local_def_id| local_def_id.to_def_id()).collect();
+    let all_def_ids: DefIdSet = tcx
+        .mir_keys(LOCAL_CRATE)
+        .iter()
+        .filter_map(|local_def_id| {
+            let def_id = local_def_id.to_def_id();
+            if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) {
+                return None;
+            }
+            Some(local_def_id.to_def_id())
+        })
+        .collect();
 
-    let (codegenned_def_ids, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
+    let codegenned_def_ids = tcx.codegened_and_inlined_items(LOCAL_CRATE);
 
-    let mut unreachable_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default();
+    let mut unused_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default();
     for &non_codegenned_def_id in all_def_ids.difference(codegenned_def_ids) {
-        // Make sure the non-codegenned (unreachable) function has a file_name
+        // Make sure the non-codegenned (unused) function has a file_name
         if let Some(non_codegenned_file_name) = tcx.covered_file_name(non_codegenned_def_id) {
-            let def_ids = unreachable_def_ids_by_file
-                .entry(*non_codegenned_file_name)
-                .or_insert_with(Vec::new);
+            let def_ids =
+                unused_def_ids_by_file.entry(*non_codegenned_file_name).or_insert_with(Vec::new);
             def_ids.push(non_codegenned_def_id);
         }
     }
 
-    if unreachable_def_ids_by_file.is_empty() {
-        // There are no unreachable functions with file names to add (in any CGU)
+    if unused_def_ids_by_file.is_empty() {
+        // There are no unused functions with file names to add (in any CGU)
         return;
     }
 
-    // Since there may be multiple `CodegenUnit`s, some codegenned_def_ids may be codegenned in a
-    // different CGU, and will be added to the function_coverage_map for each CGU. Determine which
-    // function_coverage_map has the responsibility for publishing unreachable coverage
-    // based on file name:
+    // Each `CodegenUnit` (CGU) has its own function_coverage_map, and generates a specific binary
+    // with its own coverage map.
+    //
+    // Each covered function `Instance` can be included in only one coverage map, produced from a
+    // specific function_coverage_map, from a specific CGU.
     //
-    // For each covered file name, sort ONLY the non-generic codegenned_def_ids, and if
-    // covered_def_ids.contains(the first def_id) for a given file_name, add the unreachable code
-    // region in this function_coverage_map. Otherwise, ignore it and assume another CGU's
-    // function_coverage_map will be adding it (because it will be first for one, and only one,
-    // of them).
+    // Since unused functions did not generate code, they are not associated with any CGU yet.
+    //
+    // To avoid injecting the unused functions in multiple coverage maps (for multiple CGUs)
+    // determine which function_coverage_map has the responsibility for publishing unreachable
+    // coverage, based on file name: For each unused function, find the CGU that generates the
+    // first function (based on sorted `DefId`) from the same file.
+    //
+    // Add a new `FunctionCoverage` to the `function_coverage_map`, with unreachable code regions
+    // for each region in it's MIR.
+
+    // Convert the `HashSet` of `codegenned_def_ids` to a sortable vector, and sort them.
     let mut sorted_codegenned_def_ids: Vec<DefId> =
         codegenned_def_ids.iter().map(|def_id| *def_id).collect();
     sorted_codegenned_def_ids.sort_unstable();
 
     let mut first_covered_def_id_by_file: FxHashMap<Symbol, DefId> = FxHashMap::default();
     for &def_id in sorted_codegenned_def_ids.iter() {
-        // Only consider non-generic functions, to potentially add unreachable code regions
-        if tcx.generics_of(def_id).count() == 0 {
-            if let Some(covered_file_name) = tcx.covered_file_name(def_id) {
-                // Only add files known to have unreachable functions
-                if unreachable_def_ids_by_file.contains_key(covered_file_name) {
-                    first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id);
-                }
+        if let Some(covered_file_name) = tcx.covered_file_name(def_id) {
+            // Only add files known to have unused functions
+            if unused_def_ids_by_file.contains_key(covered_file_name) {
+                first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id);
             }
         }
     }
 
     // Get the set of def_ids with coverage regions, known by *this* CoverageContext.
-    let cgu_covered_def_ids: DefIdSet =
-        function_coverage_map.keys().map(|instance| instance.def.def_id()).collect();
+    let cgu_covered_def_ids: DefIdSet = match cx.coverage_context() {
+        Some(ctx) => ctx
+            .function_coverage_map
+            .borrow()
+            .keys()
+            .map(|&instance| instance.def.def_id())
+            .collect(),
+        None => return,
+    };
 
-    let mut cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file
+    let cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file
         .iter()
         .filter_map(
             |(&file_name, def_id)| {
@@ -342,49 +344,13 @@ fn add_unreachable_coverage<'tcx>(
         )
         .collect();
 
-    // Find the first covered, non-generic function (instance) for each cgu_covered_file. Take the
-    // unreachable code regions for that file, and add them to the function.
-    //
-    // There are three `for` loops here, but (a) the lists have already been reduced to the minimum
-    // required values, the lists are further reduced (by `remove()` calls) when elements are no
-    // longer needed, and there are several opportunities to branch out of loops early.
-    for (instance, function_coverage) in function_coverage_map.iter_mut() {
-        if instance.def.attrs(tcx).len() > 0 {
-            continue;
-        }
-        // The covered function is not generic...
-        let covered_def_id = instance.def.def_id();
-        if let Some(covered_file_name) = tcx.covered_file_name(covered_def_id) {
-            if !cgu_covered_files.remove(&covered_file_name) {
-                continue;
-            }
-            // The covered function's file is one of the files with unreachable code regions, so
-            // all of the unreachable code regions for this file will be added to this function.
-            for def_id in
-                unreachable_def_ids_by_file.remove(&covered_file_name).into_iter().flatten()
-            {
-                // Note, this loop adds an unreachable code regions for each MIR-derived region.
-                // Alternatively, we could add a single code region for the maximum span of all
-                // code regions here.
-                //
-                // Observed downsides of this approach are:
-                //
-                // 1. The coverage results will appear inconsistent compared with the same (or
-                //    similar) code in a function that is reached.
-                // 2. If the function is unreachable from one crate but reachable when compiling
-                //    another referencing crate (such as a cross-crate reference to a
-                //    generic function or inlined function), actual coverage regions overlaid
-                //    on a single larger code span of `Zero` coverage can appear confusing or
-                //    wrong. Chaning the unreachable coverage from a `code_region` to a
-                //    `gap_region` can help, but still can look odd with `0` line counts for
-                //    lines between executed (> 0) lines (such as for blank lines or comments).
-                for &region in tcx.covered_code_regions(def_id) {
-                    function_coverage.add_unreachable_region(region.clone());
-                }
-            }
-            if cgu_covered_files.is_empty() {
-                break;
-            }
+    // For each file for which this CGU is responsible for adding unused function coverage,
+    // get the `def_id`s for each unused function (if any), define a synthetic function with a
+    // single LLVM coverage counter, and add the function's coverage `CodeRegion`s. to the
+    // function_coverage_map.
+    for covered_file_name in cgu_covered_files {
+        for def_id in unused_def_ids_by_file.remove(&covered_file_name).into_iter().flatten() {
+            cx.define_unused_fn(def_id);
         }
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index e47b8fde40f..afc2bdbfd52 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -1,5 +1,6 @@
 use crate::llvm;
 
+use crate::abi::{Abi, FnAbi};
 use crate::builder::Builder;
 use crate::common::CodegenCx;
 
@@ -7,33 +8,47 @@ use libc::c_uint;
 use llvm::coverageinfo::CounterMappingRegion;
 use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage};
 use rustc_codegen_ssa::traits::{
-    BaseTypeMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, MiscMethods, StaticMethods,
+    BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods,
+    MiscMethods, StaticMethods,
 };
 use rustc_data_structures::fx::FxHashMap;
+use rustc_hir as hir;
+use rustc_hir::def_id::DefId;
 use rustc_llvm::RustString;
+use rustc_middle::bug;
 use rustc_middle::mir::coverage::{
     CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op,
 };
+use rustc_middle::ty;
+use rustc_middle::ty::layout::FnAbiExt;
+use rustc_middle::ty::subst::InternalSubsts;
 use rustc_middle::ty::Instance;
 
 use std::cell::RefCell;
 use std::ffi::CString;
 
+use std::iter;
 use tracing::debug;
 
 pub mod mapgen;
 
+const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START;
+
 const VAR_ALIGN_BYTES: usize = 8;
 
 /// A context object for maintaining all state needed by the coverageinfo module.
-pub struct CrateCoverageContext<'tcx> {
+pub struct CrateCoverageContext<'ll, 'tcx> {
     // Coverage data for each instrumented function identified by DefId.
     pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>,
+    pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>,
 }
 
-impl<'tcx> CrateCoverageContext<'tcx> {
+impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> {
     pub fn new() -> Self {
-        Self { function_coverage_map: Default::default() }
+        Self {
+            function_coverage_map: Default::default(),
+            pgo_func_name_var_map: Default::default(),
+        }
     }
 
     pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> {
@@ -41,23 +56,47 @@ impl<'tcx> CrateCoverageContext<'tcx> {
     }
 }
 
-impl CoverageInfoMethods for CodegenCx<'ll, 'tcx> {
+impl CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn coverageinfo_finalize(&self) {
         mapgen::finalize(self)
     }
-}
 
-impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
-    /// Calls llvm::createPGOFuncNameVar() with the given function instance's mangled function name.
-    /// The LLVM API returns an llvm::GlobalVariable containing the function name, with the specific
-    /// variable name and linkage required by LLVM InstrProf source-based coverage instrumentation.
-    fn create_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value {
-        let llfn = self.cx.get_fn(instance);
-        let mangled_fn_name = CString::new(self.tcx.symbol_name(instance).name)
-            .expect("error converting function name to C string");
-        unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) }
+    fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value {
+        if let Some(coverage_context) = self.coverage_context() {
+            debug!("getting pgo_func_name_var for instance={:?}", instance);
+            let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut();
+            pgo_func_name_var_map
+                .entry(instance)
+                .or_insert_with(|| create_pgo_func_name_var(self, instance))
+        } else {
+            bug!("Could not get the `coverage_context`");
+        }
+    }
+
+    /// Functions with MIR-based coverage are normally codegenned _only_ if
+    /// called. LLVM coverage tools typically expect every function to be
+    /// defined (even if unused), with at least one call to LLVM intrinsic
+    /// `instrprof.increment`.
+    ///
+    /// Codegen a small function that will never be called, with one counter
+    /// that will never be incremented.
+    ///
+    /// For used/called functions, the coverageinfo was already added to the
+    /// `function_coverage_map` (keyed by function `Instance`) during codegen.
+    /// But in this case, since the unused function was _not_ previously
+    /// codegenned, collect the coverage `CodeRegion`s from the MIR and add
+    /// them. The first `CodeRegion` is used to add a single counter, with the
+    /// same counter ID used in the injected `instrprof.increment` intrinsic
+    /// call. Since the function is never called, all other `CodeRegion`s can be
+    /// added as `unreachable_region`s.
+    fn define_unused_fn(&self, def_id: DefId) {
+        let instance = declare_unused_fn(self, &def_id);
+        codegen_unused_fn_and_counter(self, instance);
+        add_unused_function_coverage(self, instance, def_id);
     }
+}
 
+impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     fn set_function_source_hash(
         &mut self,
         instance: Instance<'tcx>,
@@ -145,6 +184,100 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     }
 }
 
+fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx> {
+    let tcx = cx.tcx;
+
+    let instance = Instance::new(
+        *def_id,
+        InternalSubsts::for_item(tcx, *def_id, |param, _| {
+            if let ty::GenericParamDefKind::Lifetime = param.kind {
+                tcx.lifetimes.re_erased.into()
+            } else {
+                tcx.mk_param_from_def(param)
+            }
+        }),
+    );
+
+    let llfn = cx.declare_fn(
+        &tcx.symbol_name(instance).name,
+        &FnAbi::of_fn_ptr(
+            cx,
+            ty::Binder::dummy(tcx.mk_fn_sig(
+                iter::once(tcx.mk_unit()),
+                tcx.mk_unit(),
+                false,
+                hir::Unsafety::Unsafe,
+                Abi::Rust,
+            )),
+            &[],
+        ),
+    );
+
+    llvm::set_linkage(llfn, llvm::Linkage::WeakAnyLinkage);
+    llvm::set_visibility(llfn, llvm::Visibility::Hidden);
+
+    assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none());
+
+    instance
+}
+
+fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) {
+    let llfn = cx.get_fn(instance);
+    let mut bx = Builder::new_block(cx, llfn, "unused_function");
+    let fn_name = bx.get_pgo_func_name_var(instance);
+    let hash = bx.const_u64(0);
+    let num_counters = bx.const_u32(1);
+    let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID));
+    debug!(
+        "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?},
+            index={:?}) for unused function: {:?}",
+        fn_name, hash, num_counters, index, instance
+    );
+    bx.instrprof_increment(fn_name, hash, num_counters, index);
+    bx.ret_void();
+}
+
+fn add_unused_function_coverage(
+    cx: &CodegenCx<'ll, 'tcx>,
+    instance: Instance<'tcx>,
+    def_id: DefId,
+) {
+    let tcx = cx.tcx;
+
+    let mut function_coverage = FunctionCoverage::unused(tcx, instance);
+    for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() {
+        if index == 0 {
+            // Insert at least one real counter so the LLVM CoverageMappingReader will find expected
+            // definitions.
+            function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone());
+        } else {
+            function_coverage.add_unreachable_region(code_region.clone());
+        }
+    }
+
+    if let Some(coverage_context) = cx.coverage_context() {
+        coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage);
+    } else {
+        bug!("Could not get the `coverage_context`");
+    }
+}
+
+/// Calls llvm::createPGOFuncNameVar() with the given function instance's
+/// mangled function name. The LLVM API returns an llvm::GlobalVariable
+/// containing the function name, with the specific variable name and linkage
+/// required by LLVM InstrProf source-based coverage instrumentation. Use
+/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per
+/// `Instance`.
+fn create_pgo_func_name_var(
+    cx: &CodegenCx<'ll, 'tcx>,
+    instance: Instance<'tcx>,
+) -> &'ll llvm::Value {
+    let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name)
+        .expect("error converting function name to C string");
+    let llfn = cx.get_fn(instance);
+    unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) }
+}
+
 pub(crate) fn write_filenames_section_to_buffer<'a>(
     filenames: impl IntoIterator<Item = &'a CString>,
     buffer: &RustString,
@@ -177,6 +310,7 @@ pub(crate) fn write_mapping_to_buffer(
         );
     }
 }
+
 pub(crate) fn hash_str(strval: &str) -> u64 {
     let strval = CString::new(strval).expect("null error converting hashable str to C string");
     unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) }
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
index 7673dfb744c..c2725b83f50 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
@@ -1,4 +1,4 @@
-use super::metadata::{file_metadata, UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER};
+use super::metadata::file_metadata;
 use super::utils::DIB;
 use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext};
 use rustc_codegen_ssa::traits::*;
@@ -102,8 +102,8 @@ fn make_mir_scope(
                 DIB(cx),
                 parent_scope.dbg_scope.unwrap(),
                 file_metadata,
-                loc.line.unwrap_or(UNKNOWN_LINE_NUMBER),
-                loc.col.unwrap_or(UNKNOWN_COLUMN_NUMBER),
+                loc.line,
+                loc.col,
             )
         },
     };
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md
new file mode 100644
index 00000000000..f983d092039
--- /dev/null
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md
@@ -0,0 +1,180 @@
+# Debug Info Module
+
+This module serves the purpose of generating debug symbols. We use LLVM's
+[source level debugging](https://llvm.org/docs/SourceLevelDebugging.html)
+features for generating the debug information. The general principle is
+this:
+
+Given the right metadata in the LLVM IR, the LLVM code generator is able to
+create DWARF debug symbols for the given code. The
+[metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured
+much like DWARF *debugging information entries* (DIE), representing type
+information such as datatype layout, function signatures, block layout,
+variable location and scope information, etc. It is the purpose of this
+module to generate correct metadata and insert it into the LLVM IR.
+
+As the exact format of metadata trees may change between different LLVM
+versions, we now use LLVM
+[DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html)
+to create metadata where possible. This will hopefully ease the adaption of
+this module to future LLVM versions.
+
+The public API of the module is a set of functions that will insert the
+correct metadata into the LLVM IR when called with the right parameters.
+The module is thus driven from an outside client with functions like
+`debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`.
+
+Internally the module will try to reuse already created metadata by
+utilizing a cache. The way to get a shared metadata node when needed is
+thus to just call the corresponding function in this module:
+
+    let file_metadata = file_metadata(cx, file);
+
+The function will take care of probing the cache for an existing node for
+that exact file path.
+
+All private state used by the module is stored within either the
+CrateDebugContext struct (owned by the CodegenCx) or the
+FunctionDebugContext (owned by the FunctionCx).
+
+This file consists of three conceptual sections:
+1. The public interface of the module
+2. Module-internal metadata creation functions
+3. Minor utility functions
+
+
+## Recursive Types
+
+Some kinds of types, such as structs and enums can be recursive. That means
+that the type definition of some type X refers to some other type which in
+turn (transitively) refers to X. This introduces cycles into the type
+referral graph. A naive algorithm doing an on-demand, depth-first traversal
+of this graph when describing types, can get trapped in an endless loop
+when it reaches such a cycle.
+
+For example, the following simple type for a singly-linked list...
+
+```
+struct List {
+    value: i32,
+    tail: Option<Box<List>>,
+}
+```
+
+will generate the following callstack with a naive DFS algorithm:
+
+```
+describe(t = List)
+  describe(t = i32)
+  describe(t = Option<Box<List>>)
+    describe(t = Box<List>)
+      describe(t = List) // at the beginning again...
+      ...
+```
+
+To break cycles like these, we use "forward declarations". That is, when
+the algorithm encounters a possibly recursive type (any struct or enum), it
+immediately creates a type description node and inserts it into the cache
+*before* describing the members of the type. This type description is just
+a stub (as type members are not described and added to it yet) but it
+allows the algorithm to already refer to the type. After the stub is
+inserted into the cache, the algorithm continues as before. If it now
+encounters a recursive reference, it will hit the cache and does not try to
+describe the type anew.
+
+This behavior is encapsulated in the 'RecursiveTypeDescription' enum,
+which represents a kind of continuation, storing all state needed to
+continue traversal at the type members after the type has been registered
+with the cache. (This implementation approach might be a tad over-
+engineered and may change in the future)
+
+
+## Source Locations and Line Information
+
+In addition to data type descriptions the debugging information must also
+allow to map machine code locations back to source code locations in order
+to be useful. This functionality is also handled in this module. The
+following functions allow to control source mappings:
+
++ `set_source_location()`
++ `clear_source_location()`
++ `start_emitting_source_locations()`
+
+`set_source_location()` allows to set the current source location. All IR
+instructions created after a call to this function will be linked to the
+given source location, until another location is specified with
+`set_source_location()` or the source location is cleared with
+`clear_source_location()`. In the later case, subsequent IR instruction
+will not be linked to any source location. As you can see, this is a
+stateful API (mimicking the one in LLVM), so be careful with source
+locations set by previous calls. It's probably best to not rely on any
+specific state being present at a given point in code.
+
+One topic that deserves some extra attention is *function prologues*. At
+the beginning of a function's machine code there are typically a few
+instructions for loading argument values into allocas and checking if
+there's enough stack space for the function to execute. This *prologue* is
+not visible in the source code and LLVM puts a special PROLOGUE END marker
+into the line table at the first non-prologue instruction of the function.
+In order to find out where the prologue ends, LLVM looks for the first
+instruction in the function body that is linked to a source location. So,
+when generating prologue instructions we have to make sure that we don't
+emit source location information until the 'real' function body begins. For
+this reason, source location emission is disabled by default for any new
+function being codegened and is only activated after a call to the third
+function from the list above, `start_emitting_source_locations()`. This
+function should be called right before regularly starting to codegen the
+top-level block of the given function.
+
+There is one exception to the above rule: `llvm.dbg.declare` instruction
+must be linked to the source location of the variable being declared. For
+function parameters these `llvm.dbg.declare` instructions typically occur
+in the middle of the prologue, however, they are ignored by LLVM's prologue
+detection. The `create_argument_metadata()` and related functions take care
+of linking the `llvm.dbg.declare` instructions to the correct source
+locations even while source location emission is still disabled, so there
+is no need to do anything special with source location handling here.
+
+## Unique Type Identification
+
+In order for link-time optimization to work properly, LLVM needs a unique
+type identifier that tells it across compilation units which types are the
+same as others. This type identifier is created by
+`TypeMap::get_unique_type_id_of_type()` using the following algorithm:
+
+1. Primitive types have their name as ID
+
+2. Structs, enums and traits have a multipart identifier
+
+  1. The first part is the SVH (strict version hash) of the crate they
+     were originally defined in
+
+  2. The second part is the ast::NodeId of the definition in their
+     original crate
+
+  3. The final part is a concatenation of the type IDs of their concrete
+     type arguments if they are generic types.
+
+3. Tuple-, pointer-, and function types are structurally identified, which
+   means that they are equivalent if their component types are equivalent
+   (i.e., `(i32, i32)` is the same regardless in which crate it is used).
+
+This algorithm also provides a stable ID for types that are defined in one
+crate but instantiated from metadata within another crate. We just have to
+take care to always map crate and `NodeId`s back to the original crate
+context.
+
+As a side-effect these unique type IDs also help to solve a problem arising
+from lifetime parameters. Since lifetime parameters are completely omitted
+in debuginfo, more than one `Ty` instance may map to the same debuginfo
+type metadata, that is, some struct `Struct<'a>` may have N instantiations
+with different concrete substitutions for `'a`, and thus there will be N
+`Ty` instances for the type `Struct<'a>` even though it is not generic
+otherwise. Unfortunately this means that we cannot use `ty::type_id()` as
+cheap identifier for type metadata -- we have done this in the past, but it
+led to unnecessary metadata duplication in the best case and LLVM
+assertions in the worst. However, the unique type ID as described above
+*can* be used as identifier. Since it is comparatively expensive to
+construct, though, `ty::type_id()` is still used additionally as an
+optimization for cases where the exact same type has been seen before
+(which is most of the time).
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs b/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs
deleted file mode 100644
index 10dd5906529..00000000000
--- a/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs
+++ /dev/null
@@ -1,179 +0,0 @@
-//! # Debug Info Module
-//!
-//! This module serves the purpose of generating debug symbols. We use LLVM's
-//! [source level debugging](https://llvm.org/docs/SourceLevelDebugging.html)
-//! features for generating the debug information. The general principle is
-//! this:
-//!
-//! Given the right metadata in the LLVM IR, the LLVM code generator is able to
-//! create DWARF debug symbols for the given code. The
-//! [metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured
-//! much like DWARF *debugging information entries* (DIE), representing type
-//! information such as datatype layout, function signatures, block layout,
-//! variable location and scope information, etc. It is the purpose of this
-//! module to generate correct metadata and insert it into the LLVM IR.
-//!
-//! As the exact format of metadata trees may change between different LLVM
-//! versions, we now use LLVM
-//! [DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html)
-//! to create metadata where possible. This will hopefully ease the adaption of
-//! this module to future LLVM versions.
-//!
-//! The public API of the module is a set of functions that will insert the
-//! correct metadata into the LLVM IR when called with the right parameters.
-//! The module is thus driven from an outside client with functions like
-//! `debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`.
-//!
-//! Internally the module will try to reuse already created metadata by
-//! utilizing a cache. The way to get a shared metadata node when needed is
-//! thus to just call the corresponding function in this module:
-//!
-//!     let file_metadata = file_metadata(cx, file);
-//!
-//! The function will take care of probing the cache for an existing node for
-//! that exact file path.
-//!
-//! All private state used by the module is stored within either the
-//! CrateDebugContext struct (owned by the CodegenCx) or the
-//! FunctionDebugContext (owned by the FunctionCx).
-//!
-//! This file consists of three conceptual sections:
-//! 1. The public interface of the module
-//! 2. Module-internal metadata creation functions
-//! 3. Minor utility functions
-//!
-//!
-//! ## Recursive Types
-//!
-//! Some kinds of types, such as structs and enums can be recursive. That means
-//! that the type definition of some type X refers to some other type which in
-//! turn (transitively) refers to X. This introduces cycles into the type
-//! referral graph. A naive algorithm doing an on-demand, depth-first traversal
-//! of this graph when describing types, can get trapped in an endless loop
-//! when it reaches such a cycle.
-//!
-//! For example, the following simple type for a singly-linked list...
-//!
-//! ```
-//! struct List {
-//!     value: i32,
-//!     tail: Option<Box<List>>,
-//! }
-//! ```
-//!
-//! will generate the following callstack with a naive DFS algorithm:
-//!
-//! ```
-//! describe(t = List)
-//!   describe(t = i32)
-//!   describe(t = Option<Box<List>>)
-//!     describe(t = Box<List>)
-//!       describe(t = List) // at the beginning again...
-//!       ...
-//! ```
-//!
-//! To break cycles like these, we use "forward declarations". That is, when
-//! the algorithm encounters a possibly recursive type (any struct or enum), it
-//! immediately creates a type description node and inserts it into the cache
-//! *before* describing the members of the type. This type description is just
-//! a stub (as type members are not described and added to it yet) but it
-//! allows the algorithm to already refer to the type. After the stub is
-//! inserted into the cache, the algorithm continues as before. If it now
-//! encounters a recursive reference, it will hit the cache and does not try to
-//! describe the type anew.
-//!
-//! This behavior is encapsulated in the 'RecursiveTypeDescription' enum,
-//! which represents a kind of continuation, storing all state needed to
-//! continue traversal at the type members after the type has been registered
-//! with the cache. (This implementation approach might be a tad over-
-//! engineered and may change in the future)
-//!
-//!
-//! ## Source Locations and Line Information
-//!
-//! In addition to data type descriptions the debugging information must also
-//! allow to map machine code locations back to source code locations in order
-//! to be useful. This functionality is also handled in this module. The
-//! following functions allow to control source mappings:
-//!
-//! + set_source_location()
-//! + clear_source_location()
-//! + start_emitting_source_locations()
-//!
-//! `set_source_location()` allows to set the current source location. All IR
-//! instructions created after a call to this function will be linked to the
-//! given source location, until another location is specified with
-//! `set_source_location()` or the source location is cleared with
-//! `clear_source_location()`. In the later case, subsequent IR instruction
-//! will not be linked to any source location. As you can see, this is a
-//! stateful API (mimicking the one in LLVM), so be careful with source
-//! locations set by previous calls. It's probably best to not rely on any
-//! specific state being present at a given point in code.
-//!
-//! One topic that deserves some extra attention is *function prologues*. At
-//! the beginning of a function's machine code there are typically a few
-//! instructions for loading argument values into allocas and checking if
-//! there's enough stack space for the function to execute. This *prologue* is
-//! not visible in the source code and LLVM puts a special PROLOGUE END marker
-//! into the line table at the first non-prologue instruction of the function.
-//! In order to find out where the prologue ends, LLVM looks for the first
-//! instruction in the function body that is linked to a source location. So,
-//! when generating prologue instructions we have to make sure that we don't
-//! emit source location information until the 'real' function body begins. For
-//! this reason, source location emission is disabled by default for any new
-//! function being codegened and is only activated after a call to the third
-//! function from the list above, `start_emitting_source_locations()`. This
-//! function should be called right before regularly starting to codegen the
-//! top-level block of the given function.
-//!
-//! There is one exception to the above rule: `llvm.dbg.declare` instruction
-//! must be linked to the source location of the variable being declared. For
-//! function parameters these `llvm.dbg.declare` instructions typically occur
-//! in the middle of the prologue, however, they are ignored by LLVM's prologue
-//! detection. The `create_argument_metadata()` and related functions take care
-//! of linking the `llvm.dbg.declare` instructions to the correct source
-//! locations even while source location emission is still disabled, so there
-//! is no need to do anything special with source location handling here.
-//!
-//! ## Unique Type Identification
-//!
-//! In order for link-time optimization to work properly, LLVM needs a unique
-//! type identifier that tells it across compilation units which types are the
-//! same as others. This type identifier is created by
-//! `TypeMap::get_unique_type_id_of_type()` using the following algorithm:
-//!
-//! (1) Primitive types have their name as ID
-//! (2) Structs, enums and traits have a multipart identifier
-//!
-//!     (1) The first part is the SVH (strict version hash) of the crate they
-//!          were originally defined in
-//!
-//!     (2) The second part is the ast::NodeId of the definition in their
-//!          original crate
-//!
-//!     (3) The final part is a concatenation of the type IDs of their concrete
-//!          type arguments if they are generic types.
-//!
-//! (3) Tuple-, pointer and function types are structurally identified, which
-//!     means that they are equivalent if their component types are equivalent
-//!     (i.e., (i32, i32) is the same regardless in which crate it is used).
-//!
-//! This algorithm also provides a stable ID for types that are defined in one
-//! crate but instantiated from metadata within another crate. We just have to
-//! take care to always map crate and `NodeId`s back to the original crate
-//! context.
-//!
-//! As a side-effect these unique type IDs also help to solve a problem arising
-//! from lifetime parameters. Since lifetime parameters are completely omitted
-//! in debuginfo, more than one `Ty` instance may map to the same debuginfo
-//! type metadata, that is, some struct `Struct<'a>` may have N instantiations
-//! with different concrete substitutions for `'a`, and thus there will be N
-//! `Ty` instances for the type `Struct<'a>` even though it is not generic
-//! otherwise. Unfortunately this means that we cannot use `ty::type_id()` as
-//! cheap identifier for type metadata -- we have done this in the past, but it
-//! led to unnecessary metadata duplication in the best case and LLVM
-//! assertions in the worst. However, the unique type ID as described above
-//! *can* be used as identifier. Since it is comparatively expensive to
-//! construct, though, `ty::type_id()` is still used additionally as an
-//! optimization for cases where the exact same type has been seen before
-//! (which is most of the time).
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index b2e0a4311a7..e6fa852155b 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -1083,9 +1083,9 @@ pub fn compile_unit_metadata(
             );
         }
 
-        // Insert `llvm.ident` metadata on the wasm32 targets since that will
+        // Insert `llvm.ident` metadata on the wasm targets since that will
         // get hooked up to the "producer" sections `processed-by` information.
-        if tcx.sess.opts.target_triple.triple().starts_with("wasm32") {
+        if tcx.sess.target.is_like_wasm {
             let name_metadata = llvm::LLVMMDStringInContext(
                 debug_context.llcontext,
                 rustc_producer.as_ptr().cast(),
@@ -1842,10 +1842,7 @@ impl<'tcx> VariantInfo<'_, 'tcx> {
                     .span;
                 if !span.is_dummy() {
                     let loc = cx.lookup_debug_loc(span.lo());
-                    return Some(SourceInfo {
-                        file: file_metadata(cx, &loc.file),
-                        line: loc.line.unwrap_or(UNKNOWN_LINE_NUMBER),
-                    });
+                    return Some(SourceInfo { file: file_metadata(cx, &loc.file), line: loc.line });
                 }
             }
             _ => {}
@@ -1965,9 +1962,7 @@ fn prepare_enum_metadata(
 
     let discriminant_type_metadata = |discr: Primitive| {
         let enumerators_metadata: Vec<_> = match enum_type.kind() {
-            ty::Adt(def, _) => def
-                .discriminants(tcx)
-                .zip(&def.variants)
+            ty::Adt(def, _) => iter::zip(def.discriminants(tcx), &def.variants)
                 .map(|((_, discr), v)| {
                     let name = v.ident.as_str();
                     let is_unsigned = match discr.ty.kind() {
@@ -2339,9 +2334,7 @@ fn compute_type_parameters(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) -> &'ll DIAr
         if substs.types().next().is_some() {
             let generics = cx.tcx.generics_of(def.did);
             let names = get_parameter_names(cx, generics);
-            let template_params: Vec<_> = substs
-                .iter()
-                .zip(names)
+            let template_params: Vec<_> = iter::zip(substs, names)
                 .filter_map(|(kind, name)| {
                     if let GenericArgKind::Type(ty) = kind.unpack() {
                         let actual_type =
@@ -2484,7 +2477,7 @@ pub fn create_global_var_metadata(cx: &CodegenCx<'ll, '_>, def_id: DefId, global
         let loc = cx.lookup_debug_loc(span.lo());
         (file_metadata(cx, &loc.file), loc.line)
     } else {
-        (unknown_file_metadata(cx), None)
+        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
     };
 
     let is_local_to_unit = is_node_local_to_unit(cx, def_id);
@@ -2507,7 +2500,7 @@ pub fn create_global_var_metadata(cx: &CodegenCx<'ll, '_>, def_id: DefId, global
             linkage_name.as_ptr().cast(),
             linkage_name.len(),
             file_metadata,
-            line_number.unwrap_or(UNKNOWN_LINE_NUMBER),
+            line_number,
             type_metadata,
             is_local_to_unit,
             global,
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index c28b0d64651..e157a38aa03 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -1,5 +1,4 @@
-// See doc.rs for documentation.
-mod doc;
+#![doc = include_str!("doc.md")]
 
 use rustc_codegen_ssa::mir::debuginfo::VariableKind::*;
 
@@ -38,6 +37,7 @@ use rustc_target::abi::{LayoutOf, Primitive, Size};
 use libc::c_uint;
 use smallvec::SmallVec;
 use std::cell::RefCell;
+use std::iter;
 use tracing::debug;
 
 mod create_scope_map;
@@ -224,9 +224,9 @@ pub struct DebugLoc {
     /// Information about the original source file.
     pub file: Lrc<SourceFile>,
     /// The (1-based) line number.
-    pub line: Option<u32>,
+    pub line: u32,
     /// The (1-based) column number.
-    pub col: Option<u32>,
+    pub col: u32,
 }
 
 impl CodegenCx<'ll, '_> {
@@ -243,16 +243,16 @@ impl CodegenCx<'ll, '_> {
                 let line = (line + 1) as u32;
                 let col = (pos - line_pos).to_u32() + 1;
 
-                (file, Some(line), Some(col))
+                (file, line, col)
             }
-            Err(file) => (file, None, None),
+            Err(file) => (file, UNKNOWN_LINE_NUMBER, UNKNOWN_COLUMN_NUMBER),
         };
 
         // For MSVC, omit the column number.
         // Otherwise, emit it. This mimics clang behaviour.
         // See discussion in https://github.com/rust-lang/rust/issues/42921
         if self.sess().target.is_like_msvc {
-            DebugLoc { file, line, col: None }
+            DebugLoc { file, line, col: UNKNOWN_COLUMN_NUMBER }
         } else {
             DebugLoc { file, line, col }
         }
@@ -358,9 +358,9 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 linkage_name.as_ptr().cast(),
                 linkage_name.len(),
                 file_metadata,
-                loc.line.unwrap_or(UNKNOWN_LINE_NUMBER),
+                loc.line,
                 function_type_metadata,
-                scope_line.unwrap_or(UNKNOWN_LINE_NUMBER),
+                scope_line,
                 flags,
                 spflags,
                 maybe_definition_llfn,
@@ -449,9 +449,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             // Again, only create type information if full debuginfo is enabled
             let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full {
                 let names = get_parameter_names(cx, generics);
-                substs
-                    .iter()
-                    .zip(names)
+                iter::zip(substs, names)
                     .filter_map(|(kind, name)| {
                         if let GenericArgKind::Type(ty) = kind.unpack() {
                             let actual_type =
@@ -550,14 +548,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     ) -> &'ll DILocation {
         let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo());
 
-        unsafe {
-            llvm::LLVMRustDIBuilderCreateDebugLocation(
-                line.unwrap_or(UNKNOWN_LINE_NUMBER),
-                col.unwrap_or(UNKNOWN_COLUMN_NUMBER),
-                scope,
-                inlined_at,
-            )
-        }
+        unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) }
     }
 
     fn create_vtable_metadata(&self, ty: Ty<'tcx>, vtable: Self::Value) {
@@ -606,7 +597,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 name.as_ptr().cast(),
                 name.len(),
                 file_metadata,
-                loc.line.unwrap_or(UNKNOWN_LINE_NUMBER),
+                loc.line,
                 type_metadata,
                 true,
                 DIFlags::FlagZero,
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 668daa52ed2..fc6c1abf4af 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -334,8 +334,11 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         self.call(expect, &[cond, self.const_bool(expected)], None)
     }
 
-    fn sideeffect(&mut self, unconditional: bool) {
-        if unconditional || self.tcx.sess.opts.debugging_opts.insert_sideeffect {
+    fn sideeffect(&mut self) {
+        // This kind of check would make a ton of sense in the caller, but currently the only
+        // caller of this function is in `rustc_codegen_ssa`, which is agnostic to whether LLVM
+        // codegen backend being used, and so is unable to check the LLVM version.
+        if unsafe { llvm::LLVMRustVersionMajor() } < 12 {
             let fnname = self.get_intrinsic(&("llvm.sideeffect"));
             self.call(fnname, &[], None);
         }
@@ -390,7 +393,6 @@ fn codegen_msvc_try(
 ) {
     let llfn = get_rust_try_fn(bx, &mut |mut bx| {
         bx.set_personality_fn(bx.eh_personality());
-        bx.sideeffect(false);
 
         let mut normal = bx.build_sibling_block("normal");
         let mut catchswitch = bx.build_sibling_block("catchswitch");
@@ -552,9 +554,6 @@ fn codegen_gnu_try(
         //      (%ptr, _) = landingpad
         //      call %catch_func(%data, %ptr)
         //      ret 1
-
-        bx.sideeffect(false);
-
         let mut then = bx.build_sibling_block("then");
         let mut catch = bx.build_sibling_block("catch");
 
@@ -614,9 +613,6 @@ fn codegen_emcc_try(
         //      %catch_data[1] = %is_rust_panic
         //      call %catch_func(%data, %catch_data)
         //      ret 1
-
-        bx.sideeffect(false);
-
         let mut then = bx.build_sibling_block("then");
         let mut catch = bx.build_sibling_block("catch");
 
@@ -1057,46 +1053,48 @@ fn generic_simd_intrinsic(
         let vec_ty = bx.type_vector(elem_ty, in_len);
 
         let (intr_name, fn_ty) = match name {
-            sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
             sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
             sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
-            sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
+            sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
             _ => return_error!("unrecognized intrinsic `{}`", name),
         };
-
         let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
         let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty);
         let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
-        unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) };
         Ok(c)
     }
 
     if std::matches!(
         name,
-        sym::simd_fsqrt
-            | sym::simd_fsin
-            | sym::simd_fcos
+        sym::simd_ceil
             | sym::simd_fabs
-            | sym::simd_floor
-            | sym::simd_ceil
-            | sym::simd_fexp
+            | sym::simd_fcos
             | sym::simd_fexp2
+            | sym::simd_fexp
             | sym::simd_flog10
             | sym::simd_flog2
             | sym::simd_flog
-            | sym::simd_fpowi
-            | sym::simd_fpow
+            | sym::simd_floor
             | sym::simd_fma
+            | sym::simd_fpow
+            | sym::simd_fpowi
+            | sym::simd_fsin
+            | sym::simd_fsqrt
+            | sym::simd_round
+            | sym::simd_trunc
     ) {
         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
     }
@@ -1632,7 +1630,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
             out_elem
         );
     }
-    macro_rules! arith {
+    macro_rules! arith_binary {
         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
             $(if name == sym::$name {
                 match in_elem.kind() {
@@ -1648,7 +1646,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
             })*
         }
     }
-    arith! {
+    arith_binary! {
         simd_add: Uint, Int => add, Float => fadd;
         simd_sub: Uint, Int => sub, Float => fsub;
         simd_mul: Uint, Int => mul, Float => fmul;
@@ -1663,6 +1661,25 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
         simd_fmin: Float => minnum;
 
     }
+    macro_rules! arith_unary {
+        ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
+            $(if name == sym::$name {
+                match in_elem.kind() {
+                    $($(ty::$p(_))|* => {
+                        return Ok(bx.$call(args[0].immediate()))
+                    })*
+                    _ => {},
+                }
+                require!(false,
+                         "unsupported operation on `{}` with element `{}`",
+                         in_ty,
+                         in_elem)
+            })*
+        }
+    }
+    arith_unary! {
+        simd_neg: Int => neg, Float => fneg;
+    }
 
     if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
         let lhs = args[0].immediate();
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index d11c1592f99..5ca4b226c38 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -8,10 +8,12 @@
 #![feature(bool_to_option)]
 #![feature(const_cstr_unchecked)]
 #![feature(crate_visibility_modifier)]
+#![feature(extended_key_value_attributes)]
 #![feature(extern_types)]
 #![feature(in_band_lifetimes)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 use back::write::{create_informational_target_machine, create_target_machine};
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index e6d60044c84..32b1526f6e4 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -190,33 +190,6 @@ pub enum RealPredicate {
     RealPredicateTrue = 15,
 }
 
-impl RealPredicate {
-    pub fn from_generic(realpred: rustc_codegen_ssa::common::RealPredicate) -> Self {
-        match realpred {
-            rustc_codegen_ssa::common::RealPredicate::RealPredicateFalse => {
-                RealPredicate::RealPredicateFalse
-            }
-            rustc_codegen_ssa::common::RealPredicate::RealOEQ => RealPredicate::RealOEQ,
-            rustc_codegen_ssa::common::RealPredicate::RealOGT => RealPredicate::RealOGT,
-            rustc_codegen_ssa::common::RealPredicate::RealOGE => RealPredicate::RealOGE,
-            rustc_codegen_ssa::common::RealPredicate::RealOLT => RealPredicate::RealOLT,
-            rustc_codegen_ssa::common::RealPredicate::RealOLE => RealPredicate::RealOLE,
-            rustc_codegen_ssa::common::RealPredicate::RealONE => RealPredicate::RealONE,
-            rustc_codegen_ssa::common::RealPredicate::RealORD => RealPredicate::RealORD,
-            rustc_codegen_ssa::common::RealPredicate::RealUNO => RealPredicate::RealUNO,
-            rustc_codegen_ssa::common::RealPredicate::RealUEQ => RealPredicate::RealUEQ,
-            rustc_codegen_ssa::common::RealPredicate::RealUGT => RealPredicate::RealUGT,
-            rustc_codegen_ssa::common::RealPredicate::RealUGE => RealPredicate::RealUGE,
-            rustc_codegen_ssa::common::RealPredicate::RealULT => RealPredicate::RealULT,
-            rustc_codegen_ssa::common::RealPredicate::RealULE => RealPredicate::RealULE,
-            rustc_codegen_ssa::common::RealPredicate::RealUNE => RealPredicate::RealUNE,
-            rustc_codegen_ssa::common::RealPredicate::RealPredicateTrue => {
-                RealPredicate::RealPredicateTrue
-            }
-        }
-    }
-}
-
 /// LLVMTypeKind
 #[derive(Copy, Clone, PartialEq, Debug)]
 #[repr(C)]
@@ -711,7 +684,7 @@ pub mod coverageinfo {
     }
 
     impl CounterMappingRegion {
-        pub fn code_region(
+        crate fn code_region(
             counter: coverage_map::Counter,
             file_id: u32,
             start_line: u32,
@@ -731,7 +704,10 @@ pub mod coverageinfo {
             }
         }
 
-        pub fn expansion_region(
+        // This function might be used in the future; the LLVM API is still evolving, as is coverage
+        // support.
+        #[allow(dead_code)]
+        crate fn expansion_region(
             file_id: u32,
             expanded_file_id: u32,
             start_line: u32,
@@ -751,7 +727,10 @@ pub mod coverageinfo {
             }
         }
 
-        pub fn skipped_region(
+        // This function might be used in the future; the LLVM API is still evolving, as is coverage
+        // support.
+        #[allow(dead_code)]
+        crate fn skipped_region(
             file_id: u32,
             start_line: u32,
             start_col: u32,
@@ -770,7 +749,10 @@ pub mod coverageinfo {
             }
         }
 
-        pub fn gap_region(
+        // This function might be used in the future; the LLVM API is still evolving, as is coverage
+        // support.
+        #[allow(dead_code)]
+        crate fn gap_region(
             counter: coverage_map::Counter,
             file_id: u32,
             start_line: u32,
@@ -1031,6 +1013,7 @@ extern "C" {
     pub fn LLVMSetSection(Global: &Value, Section: *const c_char);
     pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility;
     pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility);
+    pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool);
     pub fn LLVMGetAlignment(Global: &Value) -> c_uint;
     pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint);
     pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass);
@@ -1371,7 +1354,7 @@ extern "C" {
     pub fn LLVMBuildNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
     pub fn LLVMBuildFNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
     pub fn LLVMBuildNot(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
-    pub fn LLVMRustSetHasUnsafeAlgebra(Instr: &Value);
+    pub fn LLVMRustSetFastMath(Instr: &Value);
 
     // Memory
     pub fn LLVMBuildAlloca(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value;
@@ -2145,7 +2128,13 @@ extern "C" {
     pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool;
 
     pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine);
-    pub fn LLVMRustPrintTargetFeatures(T: &TargetMachine);
+    pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t;
+    pub fn LLVMRustGetTargetFeature(
+        T: &TargetMachine,
+        Index: size_t,
+        Feature: &mut *const c_char,
+        Desc: &mut *const c_char,
+    );
 
     pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char;
     pub fn LLVMRustCreateTargetMachine(
@@ -2326,6 +2315,7 @@ extern "C" {
     pub fn LLVMRustUnsetComdat(V: &Value);
     pub fn LLVMRustSetModulePICLevel(M: &Module);
     pub fn LLVMRustSetModulePIELevel(M: &Module);
+    pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel);
     pub fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer;
     pub fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8;
     pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize;
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 544ef38c12c..b44553e4f6d 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -10,6 +10,7 @@ use rustc_span::symbol::Symbol;
 use rustc_target::spec::{MergeFunctions, PanicStrategy};
 use std::ffi::{CStr, CString};
 
+use std::ptr;
 use std::slice;
 use std::str;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -98,6 +99,9 @@ unsafe fn configure_llvm(sess: &Session) {
         // during inlining. Unfortunately these may block other optimizations.
         add("-preserve-alignment-assumptions-during-inlining=false", false);
 
+        // Use non-zero `import-instr-limit` multiplier for cold callsites.
+        add("-import-cold-multiplier=0.1", false);
+
         for arg in sess_args {
             add(&(*arg), true);
         }
@@ -189,15 +193,77 @@ pub fn print_passes() {
     }
 }
 
+fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> {
+    let len = unsafe { llvm::LLVMRustGetTargetFeaturesCount(tm) };
+    let mut ret = Vec::with_capacity(len);
+    for i in 0..len {
+        unsafe {
+            let mut feature = ptr::null();
+            let mut desc = ptr::null();
+            llvm::LLVMRustGetTargetFeature(tm, i, &mut feature, &mut desc);
+            if feature.is_null() || desc.is_null() {
+                bug!("LLVM returned a `null` target feature string");
+            }
+            let feature = CStr::from_ptr(feature).to_str().unwrap_or_else(|e| {
+                bug!("LLVM returned a non-utf8 feature string: {}", e);
+            });
+            let desc = CStr::from_ptr(desc).to_str().unwrap_or_else(|e| {
+                bug!("LLVM returned a non-utf8 feature string: {}", e);
+            });
+            ret.push((feature, desc));
+        }
+    }
+    ret
+}
+
+fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) {
+    let mut target_features = llvm_target_features(tm);
+    let mut rustc_target_features = supported_target_features(sess)
+        .iter()
+        .filter_map(|(feature, _gate)| {
+            let llvm_feature = to_llvm_feature(sess, *feature);
+            // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
+            target_features.binary_search_by_key(&llvm_feature, |(f, _d)| *f).ok().map(|index| {
+                let (_f, desc) = target_features.remove(index);
+                (*feature, desc)
+            })
+        })
+        .collect::<Vec<_>>();
+    rustc_target_features.extend_from_slice(&[(
+        "crt-static",
+        "Enables C Run-time Libraries to be statically linked",
+    )]);
+    let max_feature_len = target_features
+        .iter()
+        .chain(rustc_target_features.iter())
+        .map(|(feature, _desc)| feature.len())
+        .max()
+        .unwrap_or(0);
+
+    println!("Features supported by rustc for this target:");
+    for (feature, desc) in &rustc_target_features {
+        println!("    {1:0$} - {2}.", max_feature_len, feature, desc);
+    }
+    println!("\nCode-generation features supported by LLVM for this target:");
+    for (feature, desc) in &target_features {
+        println!("    {1:0$} - {2}.", max_feature_len, feature, desc);
+    }
+    if target_features.len() == 0 {
+        println!("    Target features listing is not supported by this LLVM version.");
+    }
+    println!("\nUse +feature to enable a feature, or -feature to disable it.");
+    println!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n");
+    println!("Code-generation features cannot be used in cfg or #[target_feature],");
+    println!("and may be renamed or removed in a future version of LLVM or rustc.\n");
+}
+
 pub(crate) fn print(req: PrintRequest, sess: &Session) {
     require_inited();
     let tm = create_informational_target_machine(sess);
-    unsafe {
-        match req {
-            PrintRequest::TargetCPUs => llvm::LLVMRustPrintTargetCPUs(tm),
-            PrintRequest::TargetFeatures => llvm::LLVMRustPrintTargetFeatures(tm),
-            _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req),
-        }
+    match req {
+        PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) },
+        PrintRequest::TargetFeatures => print_target_features(sess, tm),
+        _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req),
     }
 }
 
@@ -218,13 +284,39 @@ pub fn target_cpu(sess: &Session) -> &str {
     handle_native(name)
 }
 
-pub fn handle_native_features(sess: &Session) -> Vec<String> {
+/// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`,
+/// `--target` and similar).
+// FIXME(nagisa): Cache the output of this somehow? Maybe make this a query? We're calling this
+// for every function that has `#[target_feature]` on it. The global features won't change between
+// the functions; only crates, maybe…
+pub fn llvm_global_features(sess: &Session) -> Vec<String> {
+    // FIXME(nagisa): this should definitely be available more centrally and to other codegen backends.
+    /// These features control behaviour of rustc rather than llvm.
+    const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"];
+
+    // Features that come earlier are overriden by conflicting features later in the string.
+    // Typically we'll want more explicit settings to override the implicit ones, so:
+    //
+    // * Features from -Ctarget-cpu=*; are overriden by [^1]
+    // * Features implied by --target; are overriden by
+    // * Features from -Ctarget-feature; are overriden by
+    // * function specific features.
+    //
+    // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly
+    // through LLVM TargetMachine implementation.
+    //
+    // FIXME(nagisa): it isn't clear what's the best interaction between features implied by
+    // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always
+    // override anything that's implicit, so e.g. when there's no `--target` flag, features implied
+    // the host target are overriden by `-Ctarget-cpu=*`. On the other hand, what about when both
+    // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both
+    // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence
+    // should be taken in cases like these.
+    let mut features = vec![];
+
+    // -Ctarget-cpu=native
     match sess.opts.cg.target_cpu {
-        Some(ref s) => {
-            if s != "native" {
-                return vec![];
-            }
-
+        Some(ref s) if s == "native" => {
             let features_string = unsafe {
                 let ptr = llvm::LLVMGetHostCPUFeatures();
                 let features_string = if !ptr.is_null() {
@@ -242,11 +334,31 @@ pub fn handle_native_features(sess: &Session) -> Vec<String> {
 
                 features_string
             };
-
-            features_string.split(",").map(|s| s.to_owned()).collect()
+            features.extend(features_string.split(",").map(String::from));
         }
-        None => vec![],
-    }
+        Some(_) | None => {}
+    };
+
+    // Features implied by an implicit or explicit `--target`.
+    features.extend(
+        sess.target
+            .features
+            .split(',')
+            .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s)))
+            .map(String::from),
+    );
+
+    // -Ctarget-features
+    features.extend(
+        sess.opts
+            .cg
+            .target_feature
+            .split(',')
+            .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s)))
+            .map(String::from),
+    );
+
+    features
 }
 
 pub fn tune_cpu(sess: &Session) -> Option<&str> {
diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs
index 992e83d08fc..fc1f364e9c6 100644
--- a/compiler/rustc_codegen_llvm/src/mono_item.rs
+++ b/compiler/rustc_codegen_llvm/src/mono_item.rs
@@ -10,7 +10,9 @@ pub use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::layout::FnAbiExt;
 use rustc_middle::ty::{self, Instance, TypeFoldable};
+use rustc_session::config::CrateType;
 use rustc_target::abi::LayoutOf;
+use rustc_target::spec::RelocModel;
 use tracing::debug;
 
 impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
@@ -35,6 +37,9 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe {
             llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage));
             llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility));
+            if self.should_assume_dso_local(linkage, visibility) {
+                llvm::LLVMRustSetDSOLocal(g, true);
+            }
         }
 
         self.instances.borrow_mut().insert(instance, g);
@@ -79,6 +84,42 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
 
         attributes::from_fn_attrs(self, lldecl, instance);
 
+        unsafe {
+            if self.should_assume_dso_local(linkage, visibility) {
+                llvm::LLVMRustSetDSOLocal(lldecl, true);
+            }
+        }
+
         self.instances.borrow_mut().insert(instance, lldecl);
     }
 }
+
+impl CodegenCx<'ll, 'tcx> {
+    /// Whether a definition (NB: not declaration!) can be assumed to be local to a group of
+    /// libraries that form a single DSO or executable.
+    pub(crate) unsafe fn should_assume_dso_local(
+        &self,
+        linkage: Linkage,
+        visibility: Visibility,
+    ) -> bool {
+        if matches!(linkage, Linkage::Internal | Linkage::Private) {
+            return true;
+        }
+
+        if visibility != Visibility::Default && linkage != Linkage::ExternalWeak {
+            return true;
+        }
+
+        // Static relocation model should force copy relocations everywhere.
+        if self.tcx.sess.relocation_model() == RelocModel::Static {
+            return true;
+        }
+
+        // Symbols from executables can't really be imported any further.
+        if self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) {
+            return true;
+        }
+
+        return false;
+    }
+}
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
index 5e2f01b2c9d..12f3b52a250 100644
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -11,10 +11,9 @@ test = false
 bitflags = "1.2.1"
 cc = "1.0.1"
 itertools = "0.9"
-memmap = "0.7"
 tracing = "0.1"
 libc = "0.2.50"
-jobserver = "0.1.11"
+jobserver = "0.1.22"
 tempfile = "3.1"
 pathdiff = "0.2.0"
 
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index caa6a6a8e3a..ea75943d6f3 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -6,7 +6,7 @@ use rustc_hir::def_id::CrateNum;
 use rustc_middle::middle::cstore::{EncodedMetadata, LibSource};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_session::config::{self, CFGuard, CrateType, DebugInfo};
-use rustc_session::config::{OutputFilenames, OutputType, PrintRequest, SanitizerSet};
+use rustc_session::config::{OutputFilenames, OutputType, PrintRequest};
 use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename};
 use rustc_session::search_paths::PathKind;
 use rustc_session::utils::NativeLibKind;
@@ -16,7 +16,7 @@ use rustc_session::{filesearch, Session};
 use rustc_span::symbol::Symbol;
 use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback};
 use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo};
-use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, Target};
+use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target};
 
 use super::archive::ArchiveBuilder;
 use super::command::Command;
@@ -711,7 +711,7 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>(
             status.signal() == Some(libc::SIGILL)
         }
 
-        #[cfg(windows)]
+        #[cfg(not(unix))]
         fn is_illegal_instruction(_status: &ExitStatus) -> bool {
             false
         }
@@ -922,28 +922,20 @@ fn link_sanitizer_runtime(sess: &Session, linker: &mut dyn Linker, name: &str) {
         .map(|channel| format!("-{}", channel))
         .unwrap_or_default();
 
-    match sess.opts.target_triple.triple() {
-        "aarch64-apple-darwin" | "x86_64-apple-darwin" => {
-            // On Apple platforms, the sanitizer is always built as a dylib, and
-            // LLVM will link to `@rpath/*.dylib`, so we need to specify an
-            // rpath to the library as well (the rpath should be absolute, see
-            // PR #41352 for details).
-            let filename = format!("rustc{}_rt.{}", channel, name);
-            let path = find_sanitizer_runtime(&sess, &filename);
-            let rpath = path.to_str().expect("non-utf8 component in path");
-            linker.args(&["-Wl,-rpath", "-Xlinker", rpath]);
-            linker.link_dylib(Symbol::intern(&filename));
-        }
-        "aarch64-fuchsia"
-        | "aarch64-unknown-linux-gnu"
-        | "x86_64-fuchsia"
-        | "x86_64-unknown-freebsd"
-        | "x86_64-unknown-linux-gnu" => {
-            let filename = format!("librustc{}_rt.{}.a", channel, name);
-            let path = find_sanitizer_runtime(&sess, &filename).join(&filename);
-            linker.link_whole_rlib(&path);
-        }
-        _ => {}
+    if sess.target.is_like_osx {
+        // On Apple platforms, the sanitizer is always built as a dylib, and
+        // LLVM will link to `@rpath/*.dylib`, so we need to specify an
+        // rpath to the library as well (the rpath should be absolute, see
+        // PR #41352 for details).
+        let filename = format!("rustc{}_rt.{}", channel, name);
+        let path = find_sanitizer_runtime(&sess, &filename);
+        let rpath = path.to_str().expect("non-utf8 component in path");
+        linker.args(&["-Wl,-rpath", "-Xlinker", rpath]);
+        linker.link_dylib(Symbol::intern(&filename));
+    } else {
+        let filename = format!("librustc{}_rt.{}.a", channel, name);
+        let path = find_sanitizer_runtime(&sess, &filename).join(&filename);
+        linker.link_whole_rlib(&path);
     }
 }
 
@@ -1198,7 +1190,7 @@ fn exec_linker(
     flush_linked_file(&output, out_filename)?;
     return output;
 
-    #[cfg(unix)]
+    #[cfg(not(windows))]
     fn flush_linked_file(_: &io::Result<Output>, _: &Path) -> io::Result<()> {
         Ok(())
     }
@@ -1238,6 +1230,11 @@ fn exec_linker(
         err.raw_os_error() == Some(ERROR_FILENAME_EXCED_RANGE)
     }
 
+    #[cfg(not(any(unix, windows)))]
+    fn command_line_too_big(_: &io::Error) -> bool {
+        false
+    }
+
     struct Escape<'a> {
         arg: &'a str,
         is_like_msvc: bool,
@@ -1414,15 +1411,10 @@ fn add_link_script(cmd: &mut dyn Linker, sess: &Session, tmpdir: &Path, crate_ty
     }
 }
 
-/// Add arbitrary "user defined" args defined from command line and by `#[link_args]` attributes.
+/// Add arbitrary "user defined" args defined from command line.
 /// FIXME: Determine where exactly these args need to be inserted.
-fn add_user_defined_link_args(
-    cmd: &mut dyn Linker,
-    sess: &Session,
-    codegen_results: &CodegenResults,
-) {
+fn add_user_defined_link_args(cmd: &mut dyn Linker, sess: &Session) {
     cmd.args(&sess.opts.cg.link_args);
-    cmd.args(&*codegen_results.crate_info.link_args);
 }
 
 /// Add arbitrary "late link" args defined by the target spec.
@@ -1646,6 +1638,16 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>(
         cmd.add_eh_frame_header();
     }
 
+    // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER
+    // Make the binary compatible with data execution prevention schemes.
+    cmd.add_no_exec();
+
+    // NO-OPT-OUT, OBJECT-FILES-NO
+    // Avoid linking to dynamic libraries unless they satisfy some undefined symbols
+    // at the point at which they are specified on the command line.
+    // Must be passed before any dynamic libraries.
+    cmd.add_as_needed();
+
     // NO-OPT-OUT, OBJECT-FILES-NO
     if crt_objects_fallback {
         cmd.no_crt_objects();
@@ -1741,7 +1743,7 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>(
     );
 
     // OBJECT-FILES-NO, AUDIT-ORDER
-    if sess.opts.cg.profile_generate.enabled() || sess.opts.debugging_opts.instrument_coverage {
+    if sess.opts.cg.profile_generate.enabled() || sess.instrument_coverage() {
         cmd.pgo_gen();
     }
 
@@ -1754,7 +1756,7 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>(
     add_rpath_args(cmd, sess, codegen_results, out_filename);
 
     // OBJECT-FILES-MAYBE, CUSTOMIZATION-POINT
-    add_user_defined_link_args(cmd, sess, codegen_results);
+    add_user_defined_link_args(cmd, sess);
 
     // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER
     cmd.finalize();
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index bb35e7ec894..77d8ab49ff2 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -130,6 +130,8 @@ pub trait Linker {
     fn group_end(&mut self);
     fn linker_plugin_lto(&mut self);
     fn add_eh_frame_header(&mut self) {}
+    fn add_no_exec(&mut self) {}
+    fn add_as_needed(&mut self) {}
     fn finalize(&mut self);
 }
 
@@ -184,7 +186,7 @@ impl<'a> GccLinker<'a> {
         // * On OSX they have their own linker, not binutils'
         // * For WebAssembly the only functional linker is LLD, which doesn't
         //   support hint flags
-        !self.sess.target.is_like_osx && self.sess.target.arch != "wasm32"
+        !self.sess.target.is_like_osx && !self.sess.target.is_like_wasm
     }
 
     // Some platforms take hints about whether a library is static or dynamic.
@@ -641,6 +643,20 @@ impl<'a> Linker for GccLinker<'a> {
     fn add_eh_frame_header(&mut self) {
         self.linker_arg("--eh-frame-hdr");
     }
+
+    fn add_no_exec(&mut self) {
+        if self.sess.target.is_like_windows {
+            self.linker_arg("--nxcompat");
+        } else if self.sess.target.linker_is_gnu {
+            self.linker_arg("-znoexecstack");
+        }
+    }
+
+    fn add_as_needed(&mut self) {
+        if self.sess.target.linker_is_gnu {
+            self.linker_arg("--as-needed");
+        }
+    }
 }
 
 pub struct MsvcLinker<'a> {
@@ -878,6 +894,10 @@ impl<'a> Linker for MsvcLinker<'a> {
     fn linker_plugin_lto(&mut self) {
         // Do nothing
     }
+
+    fn add_no_exec(&mut self) {
+        self.cmd.arg("/NXCOMPAT");
+    }
 }
 
 pub struct EmLinker<'a> {
diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs
index 0d7f4447696..0ff05229466 100644
--- a/compiler/rustc_codegen_ssa/src/back/lto.rs
+++ b/compiler/rustc_codegen_ssa/src/back/lto.rs
@@ -2,6 +2,7 @@ use super::write::CodegenContext;
 use crate::traits::*;
 use crate::ModuleCodegen;
 
+use rustc_data_structures::memmap::Mmap;
 use rustc_errors::FatalError;
 
 use std::ffi::CString;
@@ -93,7 +94,7 @@ impl<B: WriteBackendMethods> LtoModuleCodegen<B> {
 pub enum SerializedModule<M: ModuleBufferMethods> {
     Local(M),
     FromRlib(Vec<u8>),
-    FromUncompressedFile(memmap::Mmap),
+    FromUncompressedFile(Mmap),
 }
 
 impl<M: ModuleBufferMethods> SerializedModule<M> {
diff --git a/compiler/rustc_codegen_ssa/src/back/rpath.rs b/compiler/rustc_codegen_ssa/src/back/rpath.rs
index 005d2efdd3b..5f21046b05e 100644
--- a/compiler/rustc_codegen_ssa/src/back/rpath.rs
+++ b/compiler/rustc_codegen_ssa/src/back/rpath.rs
@@ -24,7 +24,7 @@ pub fn get_rpath_flags(config: &mut RPathConfig<'_>) -> Vec<String> {
 
     debug!("preparing the RPATH!");
 
-    let libs = config.used_crates.clone();
+    let libs = config.used_crates;
     let libs = libs.iter().filter_map(|&(_, ref l)| l.option()).collect::<Vec<_>>();
     let rpaths = get_rpaths(config, &libs);
     let mut flags = rpaths_to_flags(&rpaths);
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 9a6f8cde1b2..b8f277c8ff5 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -15,7 +15,8 @@ use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
 use rustc_middle::ty::Instance;
 use rustc_middle::ty::{SymbolName, TyCtxt};
-use rustc_session::config::{CrateType, SanitizerSet};
+use rustc_session::config::CrateType;
+use rustc_target::spec::SanitizerSet;
 
 pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel {
     crates_export_threshold(&tcx.sess.crate_types())
@@ -188,9 +189,7 @@ fn exported_symbols_provider_local(
         }
     }
 
-    if tcx.sess.opts.debugging_opts.instrument_coverage
-        || tcx.sess.opts.cg.profile_generate.enabled()
-    {
+    if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() {
         // These are weak symbols that point to the profile version and the
         // profile name, which need to be treated as exported so LTO doesn't nix
         // them.
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 7b8ce157fc2..c8688faa80b 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -10,6 +10,7 @@ use crate::{
 use crate::traits::*;
 use jobserver::{Acquired, Client};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::profiling::TimingGuard;
 use rustc_data_structures::profiling::VerboseTimingGuard;
@@ -27,12 +28,12 @@ use rustc_middle::middle::exported_symbols::SymbolExportLevel;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::cgu_reuse_tracker::CguReuseTracker;
 use rustc_session::config::{self, CrateType, Lto, OutputFilenames, OutputType};
-use rustc_session::config::{Passes, SanitizerSet, SwitchWithOptPath};
+use rustc_session::config::{Passes, SwitchWithOptPath};
 use rustc_session::Session;
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{BytePos, FileName, InnerSpan, Pos, Span};
-use rustc_target::spec::{MergeFunctions, PanicStrategy};
+use rustc_target::spec::{MergeFunctions, PanicStrategy, SanitizerSet};
 
 use std::any::Any;
 use std::fs;
@@ -106,7 +107,7 @@ pub struct ModuleConfig {
     pub vectorize_loop: bool,
     pub vectorize_slp: bool,
     pub merge_functions: bool,
-    pub inline_threshold: Option<usize>,
+    pub inline_threshold: Option<u32>,
     pub new_llvm_pass_manager: bool,
     pub emit_lifetime_markers: bool,
 }
@@ -176,7 +177,7 @@ impl ModuleConfig {
 
                     // The rustc option `-Zinstrument_coverage` injects intrinsic calls to
                     // `llvm.instrprof.increment()`, which requires the LLVM `instrprof` pass.
-                    if sess.opts.debugging_opts.instrument_coverage {
+                    if sess.instrument_coverage() {
                         passes.push("instrprof".to_owned());
                     }
                     passes
@@ -433,12 +434,10 @@ pub fn start_async_codegen<B: ExtraBackendMethods>(
     let sess = tcx.sess;
 
     let crate_name = tcx.crate_name(LOCAL_CRATE);
-    let no_builtins = tcx.sess.contains_name(&tcx.hir().krate().item.attrs, sym::no_builtins);
-    let is_compiler_builtins =
-        tcx.sess.contains_name(&tcx.hir().krate().item.attrs, sym::compiler_builtins);
-    let subsystem = tcx
-        .sess
-        .first_attr_value_str_by_name(&tcx.hir().krate().item.attrs, sym::windows_subsystem);
+    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
+    let no_builtins = tcx.sess.contains_name(crate_attrs, sym::no_builtins);
+    let is_compiler_builtins = tcx.sess.contains_name(crate_attrs, sym::compiler_builtins);
+    let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
     let windows_subsystem = subsystem.map(|subsystem| {
         if subsystem != sym::windows && subsystem != sym::console {
             tcx.sess.fatal(&format!(
@@ -1095,7 +1094,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
     //   only place where we have access to the compiler `Session`.
     // - LLVM work can be done on any thread.
     // - Codegen can only happen on the main thread.
-    // - Each thread doing substantial work most be in possession of a `Token`
+    // - Each thread doing substantial work must be in possession of a `Token`
     //   from the `Jobserver`.
     // - The compiler process always holds one `Token`. Any additional `Tokens`
     //   have to be requested from the `Jobserver`.
@@ -1147,7 +1146,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
     // if possible. These two goals are at odds with each other: If memory
     // consumption were not an issue, we could just let the main thread produce
     // LLVM WorkItems at full speed, assuring maximal utilization of
-    // Tokens/LLVM worker threads. However, since codegen usual is faster
+    // Tokens/LLVM worker threads. However, since codegen is usually faster
     // than LLVM processing, the queue of LLVM WorkItems would fill up and each
     // WorkItem potentially holds on to a substantial amount of memory.
     //
@@ -1960,7 +1959,7 @@ pub fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
         .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
 
     let mmap = unsafe {
-        memmap::Mmap::map(&file).unwrap_or_else(|e| {
+        Mmap::map(file).unwrap_or_else(|e| {
             panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
         })
     };
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 08e31c3b37f..318eed76acf 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -754,7 +754,6 @@ impl CrateInfo {
             is_no_builtins: Default::default(),
             native_libraries: Default::default(),
             used_libraries: tcx.native_libraries(LOCAL_CRATE).iter().map(Into::into).collect(),
-            link_args: tcx.link_args(LOCAL_CRATE),
             crate_name: Default::default(),
             used_crates_dynamic: cstore::used_crates(tcx, LinkagePreference::RequireDynamic),
             used_crates_static: cstore::used_crates(tcx, LinkagePreference::RequireStatic),
diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs
index af6c476292b..962c01c2ee7 100644
--- a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs
+++ b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs
@@ -24,21 +24,39 @@ pub enum CounterKind {
 pub struct Counter {
     // Important: The layout (order and types of fields) must match its C++ counterpart.
     pub kind: CounterKind,
-    pub id: u32,
+    id: u32,
 }
 
 impl Counter {
+    /// Constructs a new `Counter` of kind `Zero`. For this `CounterKind`, the
+    /// `id` is not used.
     pub fn zero() -> Self {
         Self { kind: CounterKind::Zero, id: 0 }
     }
 
+    /// Constructs a new `Counter` of kind `CounterValueReference`, and converts
+    /// the given 1-based counter_id to the required 0-based equivalent for
+    /// the `Counter` encoding.
     pub fn counter_value_reference(counter_id: CounterValueReference) -> Self {
-        Self { kind: CounterKind::CounterValueReference, id: counter_id.into() }
+        Self { kind: CounterKind::CounterValueReference, id: counter_id.zero_based_index() }
     }
 
+    /// Constructs a new `Counter` of kind `Expression`.
     pub fn expression(mapped_expression_index: MappedExpressionIndex) -> Self {
         Self { kind: CounterKind::Expression, id: mapped_expression_index.into() }
     }
+
+    /// Returns true if the `Counter` kind is `Zero`.
+    pub fn is_zero(&self) -> bool {
+        matches!(self.kind, CounterKind::Zero)
+    }
+
+    /// An explicitly-named function to get the ID value, making it more obvious
+    /// that the stored value is now 0-based.
+    pub fn zero_based_id(&self) -> u32 {
+        debug_assert!(!self.is_zero(), "`id` is undefined for CounterKind::Zero");
+        self.id
+    }
 }
 
 /// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L147)
diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs
index 549b8d41f51..4458fd68678 100644
--- a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs
+++ b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs
@@ -8,7 +8,7 @@ use rustc_middle::mir::coverage::{
 use rustc_middle::ty::Instance;
 use rustc_middle::ty::TyCtxt;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct Expression {
     lhs: ExpressionOperandId,
     op: Op,
@@ -31,27 +31,44 @@ pub struct Expression {
 pub struct FunctionCoverage<'tcx> {
     instance: Instance<'tcx>,
     source_hash: u64,
+    is_used: bool,
     counters: IndexVec<CounterValueReference, Option<CodeRegion>>,
     expressions: IndexVec<InjectedExpressionIndex, Option<Expression>>,
     unreachable_regions: Vec<CodeRegion>,
 }
 
 impl<'tcx> FunctionCoverage<'tcx> {
+    /// Creates a new set of coverage data for a used (called) function.
     pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
+        Self::create(tcx, instance, true)
+    }
+
+    /// Creates a new set of coverage data for an unused (never called) function.
+    pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
+        Self::create(tcx, instance, false)
+    }
+
+    fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self {
         let coverageinfo = tcx.coverageinfo(instance.def_id());
         debug!(
-            "FunctionCoverage::new(instance={:?}) has coverageinfo={:?}",
-            instance, coverageinfo
+            "FunctionCoverage::new(instance={:?}) has coverageinfo={:?}. is_used={}",
+            instance, coverageinfo, is_used
         );
         Self {
             instance,
             source_hash: 0, // will be set with the first `add_counter()`
+            is_used,
             counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize),
             expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize),
             unreachable_regions: Vec::new(),
         }
     }
 
+    /// Returns true for a used (called) function, and false for an unused function.
+    pub fn is_used(&self) -> bool {
+        self.is_used
+    }
+
     /// Sets the function source hash value. If called multiple times for the same function, all
     /// calls should have the same hash value.
     pub fn set_function_source_hash(&mut self, source_hash: u64) {
@@ -64,7 +81,9 @@ impl<'tcx> FunctionCoverage<'tcx> {
 
     /// Adds a code region to be counted by an injected counter intrinsic.
     pub fn add_counter(&mut self, id: CounterValueReference, region: CodeRegion) {
-        self.counters[id].replace(region).expect_none("add_counter called with duplicate `id`");
+        if let Some(previous_region) = self.counters[id].replace(region.clone()) {
+            assert_eq!(previous_region, region, "add_counter: code region for id changed");
+        }
     }
 
     /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other
@@ -94,9 +113,18 @@ impl<'tcx> FunctionCoverage<'tcx> {
             expression_id, lhs, op, rhs, region
         );
         let expression_index = self.expression_index(u32::from(expression_id));
-        self.expressions[expression_index]
-            .replace(Expression { lhs, op, rhs, region })
-            .expect_none("add_counter_expression called with duplicate `id_descending_from_max`");
+        if let Some(previous_expression) = self.expressions[expression_index].replace(Expression {
+            lhs,
+            op,
+            rhs,
+            region: region.clone(),
+        }) {
+            assert_eq!(
+                previous_expression,
+                Expression { lhs, op, rhs, region },
+                "add_counter_expression: expression for id changed"
+            );
+        }
     }
 
     /// Add a region that will be marked as "unreachable", with a constant "zero counter".
@@ -117,8 +145,8 @@ impl<'tcx> FunctionCoverage<'tcx> {
         &'a self,
     ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &'a CodeRegion)>) {
         assert!(
-            self.source_hash != 0,
-            "No counters provided the source_hash for function: {:?}",
+            self.source_hash != 0 || !self.is_used,
+            "No counters provided the source_hash for used function: {:?}",
             self.instance
         );
 
@@ -135,9 +163,7 @@ impl<'tcx> FunctionCoverage<'tcx> {
         self.counters.iter_enumerated().filter_map(|(index, entry)| {
             // Option::map() will return None to filter out missing counters. This may happen
             // if, for example, a MIR-instrumented counter is removed during an optimization.
-            entry.as_ref().map(|region| {
-                (Counter::counter_value_reference(index as CounterValueReference), region)
-            })
+            entry.as_ref().map(|region| (Counter::counter_value_reference(index), region))
         })
     }
 
@@ -178,9 +204,15 @@ impl<'tcx> FunctionCoverage<'tcx> {
             if id == ExpressionOperandId::ZERO {
                 Some(Counter::zero())
             } else if id.index() < self.counters.len() {
+                debug_assert!(
+                    id.index() > 0,
+                    "ExpressionOperandId indexes for counters are 1-based, but this id={}",
+                    id.index()
+                );
                 // Note: Some codegen-injected Counters may be only referenced by `Expression`s,
                 // and may not have their own `CodeRegion`s,
                 let index = CounterValueReference::from(id.index());
+                // Note, the conversion to LLVM `Counter` adjusts the index to be zero-based.
                 Some(Counter::counter_value_reference(index))
             } else {
                 let index = self.expression_index(u32::from(id));
@@ -205,19 +237,60 @@ impl<'tcx> FunctionCoverage<'tcx> {
             let optional_region = &expression.region;
             let Expression { lhs, op, rhs, .. } = *expression;
 
-            if let Some(Some((lhs_counter, rhs_counter))) =
-                id_to_counter(&new_indexes, lhs).map(|lhs_counter| {
+            if let Some(Some((lhs_counter, mut rhs_counter))) = id_to_counter(&new_indexes, lhs)
+                .map(|lhs_counter| {
                     id_to_counter(&new_indexes, rhs).map(|rhs_counter| (lhs_counter, rhs_counter))
                 })
             {
+                if lhs_counter.is_zero() && op.is_subtract() {
+                    // The left side of a subtraction was probably optimized out. As an example,
+                    // a branch condition might be evaluated as a constant expression, and the
+                    // branch could be removed, dropping unused counters in the process.
+                    //
+                    // Since counters are unsigned, we must assume the result of the expression
+                    // can be no more and no less than zero. An expression known to evaluate to zero
+                    // does not need to be added to the coverage map.
+                    //
+                    // Coverage test `loops_branches.rs` includes multiple variations of branches
+                    // based on constant conditional (literal `true` or `false`), and demonstrates
+                    // that the expected counts are still correct.
+                    debug!(
+                        "Expression subtracts from zero (assume unreachable): \
+                        original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}",
+                        original_index, lhs, op, rhs, optional_region,
+                    );
+                    rhs_counter = Counter::zero();
+                }
                 debug_assert!(
-                    (lhs_counter.id as usize)
-                        < usize::max(self.counters.len(), self.expressions.len())
+                    lhs_counter.is_zero()
+                        // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16`
+                        || ((lhs_counter.zero_based_id() as usize)
+                            <= usize::max(self.counters.len(), self.expressions.len())),
+                    "lhs id={} > both counters.len()={} and expressions.len()={}
+                    ({:?} {:?} {:?})",
+                    lhs_counter.zero_based_id(),
+                    self.counters.len(),
+                    self.expressions.len(),
+                    lhs_counter,
+                    op,
+                    rhs_counter,
                 );
+
                 debug_assert!(
-                    (rhs_counter.id as usize)
-                        < usize::max(self.counters.len(), self.expressions.len())
+                    rhs_counter.is_zero()
+                        // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16`
+                        || ((rhs_counter.zero_based_id() as usize)
+                            <= usize::max(self.counters.len(), self.expressions.len())),
+                    "rhs id={} > both counters.len()={} and expressions.len()={}
+                    ({:?} {:?} {:?})",
+                    rhs_counter.zero_based_id(),
+                    self.counters.len(),
+                    self.expressions.len(),
+                    lhs_counter,
+                    op,
+                    rhs_counter,
                 );
+
                 // Both operands exist. `Expression` operands exist in `self.expressions` and have
                 // been assigned a `new_index`.
                 let mapped_expression_index =
@@ -240,11 +313,15 @@ impl<'tcx> FunctionCoverage<'tcx> {
                     expression_regions.push((Counter::expression(mapped_expression_index), region));
                 }
             } else {
-                debug!(
-                    "Ignoring expression with one or more missing operands: \
-                    original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}",
-                    original_index, lhs, op, rhs, optional_region,
-                )
+                bug!(
+                    "expression has one or more missing operands \
+                      original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}",
+                    original_index,
+                    lhs,
+                    op,
+                    rhs,
+                    optional_region,
+                );
             }
         }
         (counter_expressions, expression_regions.into_iter())
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 0307117e1c8..f0f45b067b3 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -1,14 +1,16 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(assert_matches)]
 #![feature(bool_to_option)]
-#![feature(option_expect_none)]
 #![feature(box_patterns)]
 #![feature(drain_filter)]
 #![feature(try_blocks)]
 #![feature(in_band_lifetimes)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(associated_type_bounds)]
+#![feature(iter_zip)]
 #![recursion_limit = "256"]
+#![feature(box_syntax)]
 
 //! This crate contains codegen code that is used by all codegen backends (LLVM and others).
 //! The backend-agnostic functions of this crate use functions defined in various traits that
@@ -137,7 +139,6 @@ pub struct CrateInfo {
     pub native_libraries: FxHashMap<CrateNum, Vec<NativeLib>>,
     pub crate_name: FxHashMap<CrateNum, String>,
     pub used_libraries: Vec<NativeLib>,
-    pub link_args: Lrc<Vec<String>>,
     pub used_crate_source: FxHashMap<CrateNum, Lrc<CrateSource>>,
     pub used_crates_static: Vec<(CrateNum, LibSource)>,
     pub used_crates_dynamic: Vec<(CrateNum, LibSource)>,
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index 289629d9215..38e928145a8 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -231,7 +231,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
     fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) {
         let check = match terminator.kind {
             mir::TerminatorKind::Call { func: mir::Operand::Constant(ref c), ref args, .. } => {
-                match *c.literal.ty.kind() {
+                match *c.ty().kind() {
                     ty::FnDef(did, _) => Some((did, args)),
                     _ => None,
                 }
@@ -281,7 +281,18 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
                     Some(assignment_location) => {
                         assignment_location.dominates(location, &self.dominators)
                     }
-                    None => false,
+                    None => {
+                        debug!("No first assignment found for {:?}", local);
+                        // We have not seen any assignment to the local yet,
+                        // but before marking not_ssa, check if it is a ZST,
+                        // in which case we don't need to initialize the local.
+                        let ty = self.fx.mir.local_decls[local].ty;
+                        let ty = self.fx.monomorphize(ty);
+
+                        let is_zst = self.fx.cx.layout_of(ty).is_zst();
+                        debug!("is_zst: {}", is_zst);
+                        is_zst
+                    }
                 };
                 if !ssa_read {
                     self.not_ssa(local);
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 9ce90669800..fd3f89a2aee 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -146,24 +146,6 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             }
         }
     }
-
-    // Generate sideeffect intrinsic if jumping to any of the targets can form
-    // a loop.
-    fn maybe_sideeffect<Bx: BuilderMethods<'a, 'tcx>>(
-        &self,
-        mir: &'tcx mir::Body<'tcx>,
-        bx: &mut Bx,
-        targets: &[mir::BasicBlock],
-    ) {
-        if bx.tcx().sess.opts.debugging_opts.insert_sideeffect {
-            if targets.iter().any(|&target| {
-                target <= self.bb
-                    && target.start_location().is_predecessor_of(self.bb.start_location(), mir)
-            }) {
-                bx.sideeffect(false);
-            }
-        }
-    }
 }
 
 /// Codegen implementations for some terminator variants.
@@ -198,8 +180,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let discr = self.codegen_operand(&mut bx, &discr);
         // `switch_ty` is redundant, sanity-check that.
         assert_eq!(discr.layout.ty, switch_ty);
-        helper.maybe_sideeffect(self.mir, &mut bx, targets.all_targets());
-
         let mut target_iter = targets.iter();
         if target_iter.len() == 1 {
             // If there are two targets (one conditional, one fallback), emit br instead of switch
@@ -308,7 +288,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def {
             // we don't actually need to drop anything.
-            helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
             helper.funclet_br(self, &mut bx, target);
             return;
         }
@@ -337,7 +316,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
             _ => (bx.get_fn_addr(drop_fn), FnAbi::of_instance(&bx, drop_fn, &[])),
         };
-        helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
         helper.do_call(
             self,
             &mut bx,
@@ -379,7 +357,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         // Don't codegen the panic block if success if known.
         if const_cond == Some(expected) {
-            helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
             helper.funclet_br(self, &mut bx, target);
             return;
         }
@@ -390,7 +367,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // Create the failure block and the conditional branch to it.
         let lltarget = helper.llblock(self, target);
         let panic_block = self.new_block("panic");
-        helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
         if expected {
             bx.cond_br(cond, lltarget, panic_block.llbb());
         } else {
@@ -491,9 +467,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let fn_abi = FnAbi::of_instance(bx, instance, &[]);
                 let llfn = bx.get_fn_addr(instance);
 
-                if let Some((_, target)) = destination.as_ref() {
-                    helper.maybe_sideeffect(self.mir, bx, &[*target]);
-                }
                 // Codegen the actual panic invoke/call.
                 helper.do_call(
                     self,
@@ -507,7 +480,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             } else {
                 // a NOP
                 let target = destination.as_ref().unwrap().1;
-                helper.maybe_sideeffect(self.mir, bx, &[target]);
                 helper.funclet_br(self, bx, target)
             }
             true
@@ -551,7 +523,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         if let Some(ty::InstanceDef::DropGlue(_, None)) = def {
             // Empty drop glue; a no-op.
             let &(_, target) = destination.as_ref().unwrap();
-            helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
             helper.funclet_br(self, &mut bx, target);
             return;
         }
@@ -586,7 +557,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             if let Some(destination_ref) = destination.as_ref() {
                 let &(dest, target) = destination_ref;
                 self.codegen_transmute(&mut bx, &args[0], dest);
-                helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
                 helper.funclet_br(self, &mut bx, target);
             } else {
                 // If we are trying to transmute to an uninhabited type,
@@ -634,74 +604,73 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     location.val.store(&mut bx, tmp);
                 }
                 self.store_return(&mut bx, ret_dest, &fn_abi.ret, location.immediate());
-
-                helper.maybe_sideeffect(self.mir, &mut bx, &[*target]);
                 helper.funclet_br(self, &mut bx, *target);
             }
             return;
         }
 
-        if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) {
-            let intrinsic = intrinsic.unwrap();
-            let dest = match ret_dest {
-                _ if fn_abi.ret.is_indirect() => llargs[0],
-                ReturnDest::Nothing => {
-                    bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
-                }
-                ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
-                ReturnDest::DirectOperand(_) => {
-                    bug!("Cannot use direct operand with an intrinsic call")
-                }
-            };
+        match intrinsic {
+            None | Some(sym::drop_in_place) => {}
+            Some(sym::copy_nonoverlapping) => unreachable!(),
+            Some(intrinsic) => {
+                let dest = match ret_dest {
+                    _ if fn_abi.ret.is_indirect() => llargs[0],
+                    ReturnDest::Nothing => {
+                        bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
+                    }
+                    ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
+                    ReturnDest::DirectOperand(_) => {
+                        bug!("Cannot use direct operand with an intrinsic call")
+                    }
+                };
 
-            let args: Vec<_> = args
-                .iter()
-                .enumerate()
-                .map(|(i, arg)| {
-                    // The indices passed to simd_shuffle* in the
-                    // third argument must be constant. This is
-                    // checked by const-qualification, which also
-                    // promotes any complex rvalues to constants.
-                    if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
-                        if let mir::Operand::Constant(constant) = arg {
-                            let c = self.eval_mir_constant(constant);
-                            let (llval, ty) = self.simd_shuffle_indices(
-                                &bx,
-                                constant.span,
-                                constant.literal.ty,
-                                c,
-                            );
-                            return OperandRef { val: Immediate(llval), layout: bx.layout_of(ty) };
-                        } else {
-                            span_bug!(span, "shuffle indices must be constant");
+                let args: Vec<_> = args
+                    .iter()
+                    .enumerate()
+                    .map(|(i, arg)| {
+                        // The indices passed to simd_shuffle* in the
+                        // third argument must be constant. This is
+                        // checked by const-qualification, which also
+                        // promotes any complex rvalues to constants.
+                        if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
+                            if let mir::Operand::Constant(constant) = arg {
+                                let c = self.eval_mir_constant(constant);
+                                let (llval, ty) =
+                                    self.simd_shuffle_indices(&bx, constant.span, constant.ty(), c);
+                                return OperandRef {
+                                    val: Immediate(llval),
+                                    layout: bx.layout_of(ty),
+                                };
+                            } else {
+                                span_bug!(span, "shuffle indices must be constant");
+                            }
                         }
-                    }
 
-                    self.codegen_operand(&mut bx, arg)
-                })
-                .collect();
+                        self.codegen_operand(&mut bx, arg)
+                    })
+                    .collect();
+
+                Self::codegen_intrinsic_call(
+                    &mut bx,
+                    *instance.as_ref().unwrap(),
+                    &fn_abi,
+                    &args,
+                    dest,
+                    span,
+                );
 
-            Self::codegen_intrinsic_call(
-                &mut bx,
-                *instance.as_ref().unwrap(),
-                &fn_abi,
-                &args,
-                dest,
-                span,
-            );
+                if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
+                    self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
+                }
 
-            if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
-                self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
-            }
+                if let Some((_, target)) = *destination {
+                    helper.funclet_br(self, &mut bx, target);
+                } else {
+                    bx.unreachable();
+                }
 
-            if let Some((_, target)) = *destination {
-                helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
-                helper.funclet_br(self, &mut bx, target);
-            } else {
-                bx.unreachable();
+                return;
             }
-
-            return;
         }
 
         // Split the rust-call tupled arguments off.
@@ -811,9 +780,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             _ => span_bug!(span, "no llfn for call"),
         };
 
-        if let Some((_, target)) = destination.as_ref() {
-            helper.maybe_sideeffect(self.mir, &mut bx, &[*target]);
-        }
         helper.do_call(
             self,
             &mut bx,
@@ -856,45 +822,41 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     InlineAsmOperandRef::InOut { reg, late, in_value, out_place }
                 }
                 mir::InlineAsmOperand::Const { ref value } => {
-                    if let mir::Operand::Constant(constant) = value {
-                        let const_value = self
-                            .eval_mir_constant(constant)
-                            .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved"));
-                        let ty = constant.literal.ty;
-                        let size = bx.layout_of(ty).size;
-                        let scalar = match const_value {
-                            ConstValue::Scalar(s) => s,
-                            _ => span_bug!(
-                                span,
-                                "expected Scalar for promoted asm const, but got {:#?}",
-                                const_value
-                            ),
-                        };
-                        let value = scalar.assert_bits(size);
-                        let string = match ty.kind() {
-                            ty::Uint(_) => value.to_string(),
-                            ty::Int(int_ty) => {
-                                match int_ty.normalize(bx.tcx().sess.target.pointer_width) {
-                                    ty::IntTy::I8 => (value as i8).to_string(),
-                                    ty::IntTy::I16 => (value as i16).to_string(),
-                                    ty::IntTy::I32 => (value as i32).to_string(),
-                                    ty::IntTy::I64 => (value as i64).to_string(),
-                                    ty::IntTy::I128 => (value as i128).to_string(),
-                                    ty::IntTy::Isize => unreachable!(),
-                                }
+                    let const_value = self
+                        .eval_mir_constant(value)
+                        .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved"));
+                    let ty = value.ty();
+                    let size = bx.layout_of(ty).size;
+                    let scalar = match const_value {
+                        ConstValue::Scalar(s) => s,
+                        _ => span_bug!(
+                            span,
+                            "expected Scalar for promoted asm const, but got {:#?}",
+                            const_value
+                        ),
+                    };
+                    let value = scalar.assert_bits(size);
+                    let string = match ty.kind() {
+                        ty::Uint(_) => value.to_string(),
+                        ty::Int(int_ty) => {
+                            match int_ty.normalize(bx.tcx().sess.target.pointer_width) {
+                                ty::IntTy::I8 => (value as i8).to_string(),
+                                ty::IntTy::I16 => (value as i16).to_string(),
+                                ty::IntTy::I32 => (value as i32).to_string(),
+                                ty::IntTy::I64 => (value as i64).to_string(),
+                                ty::IntTy::I128 => (value as i128).to_string(),
+                                ty::IntTy::Isize => unreachable!(),
                             }
-                            ty::Float(ty::FloatTy::F32) => f32::from_bits(value as u32).to_string(),
-                            ty::Float(ty::FloatTy::F64) => f64::from_bits(value as u64).to_string(),
-                            _ => span_bug!(span, "asm const has bad type {}", ty),
-                        };
-                        InlineAsmOperandRef::Const { string }
-                    } else {
-                        span_bug!(span, "asm const is not a constant");
-                    }
+                        }
+                        ty::Float(ty::FloatTy::F32) => f32::from_bits(value as u32).to_string(),
+                        ty::Float(ty::FloatTy::F64) => f64::from_bits(value as u64).to_string(),
+                        _ => span_bug!(span, "asm const has bad type {}", ty),
+                    };
+                    InlineAsmOperandRef::Const { string }
                 }
                 mir::InlineAsmOperand::SymFn { ref value } => {
                     let literal = self.monomorphize(value.literal);
-                    if let ty::FnDef(def_id, substs) = *literal.ty.kind() {
+                    if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
                         let instance = ty::Instance::resolve_for_fn_ptr(
                             bx.tcx(),
                             ty::ParamEnv::reveal_all(),
@@ -963,22 +925,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             mir::TerminatorKind::Goto { target } => {
                 if bb == target {
-                    // This is an unconditional branch back to this same basic
-                    // block. That means we have something like a `loop {}`
-                    // statement. Currently LLVM miscompiles this because it
-                    // assumes forward progress. We want to prevent this in all
-                    // cases, but that has a fairly high cost to compile times
-                    // currently. Instead, try to handle this specific case
-                    // which comes up commonly in practice (e.g., in embedded
-                    // code).
+                    // This is an unconditional branch back to this same basic block. That means we
+                    // have something like a `loop {}` statement. LLVM versions before 12.0
+                    // miscompile this because they assume forward progress. For older versions
+                    // try to handle just this specific case which comes up commonly in practice
+                    // (e.g., in embedded code).
                     //
-                    // The `true` here means we insert side effects regardless
-                    // of -Zinsert-sideeffect being passed on unconditional
-                    // branching to the same basic block.
-                    bx.sideeffect(true);
-                } else {
-                    helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+                    // NB: the `sideeffect` currently checks for the LLVM version used internally.
+                    bx.sideeffect();
                 }
+
                 helper.funclet_br(self, &mut bx, target);
             }
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
index b79a221a0e7..fa8a53e60b1 100644
--- a/compiler/rustc_codegen_ssa/src/mir/constant.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -16,7 +16,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         constant: &mir::Constant<'tcx>,
     ) -> Result<OperandRef<'tcx, Bx::Value>, ErrorHandled> {
         let val = self.eval_mir_constant(constant)?;
-        let ty = self.monomorphize(constant.literal.ty);
+        let ty = self.monomorphize(constant.ty());
         Ok(OperandRef::from_const(bx, val, ty))
     }
 
@@ -24,11 +24,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &self,
         constant: &mir::Constant<'tcx>,
     ) -> Result<ConstValue<'tcx>, ErrorHandled> {
-        match self.monomorphize(constant.literal).val {
-            ty::ConstKind::Unevaluated(def, substs, promoted) => self
+        let ct = self.monomorphize(constant.literal);
+        let ct = match ct {
+            mir::ConstantKind::Ty(ct) => ct,
+            mir::ConstantKind::Val(val, _) => return Ok(val),
+        };
+        match ct.val {
+            ty::ConstKind::Unevaluated(ct) => self
                 .cx
                 .tcx()
-                .const_eval_resolve(ty::ParamEnv::reveal_all(), def, substs, promoted, None)
+                .const_eval_resolve(ty::ParamEnv::reveal_all(), ct, None)
                 .map_err(|err| {
                     self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
                     err
diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs
index a115d358666..621ec0519c9 100644
--- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs
@@ -2,30 +2,41 @@ use crate::traits::*;
 
 use rustc_middle::mir::coverage::*;
 use rustc_middle::mir::Coverage;
+use rustc_middle::mir::SourceScope;
 
 use super::FunctionCx;
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
-    pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage) {
+    pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) {
+        // Determine the instance that coverage data was originally generated for.
+        let scope_data = &self.mir.source_scopes[scope];
+        let instance = if let Some((inlined_instance, _)) = scope_data.inlined {
+            self.monomorphize(inlined_instance)
+        } else if let Some(inlined_scope) = scope_data.inlined_parent_scope {
+            self.monomorphize(self.mir.source_scopes[inlined_scope].inlined.unwrap().0)
+        } else {
+            self.instance
+        };
+
         let Coverage { kind, code_region } = coverage;
         match kind {
             CoverageKind::Counter { function_source_hash, id } => {
-                if bx.set_function_source_hash(self.instance, function_source_hash) {
+                if bx.set_function_source_hash(instance, function_source_hash) {
                     // If `set_function_source_hash()` returned true, the coverage map is enabled,
                     // so continue adding the counter.
                     if let Some(code_region) = code_region {
                         // Note: Some counters do not have code regions, but may still be referenced
                         // from expressions. In that case, don't add the counter to the coverage map,
                         // but do inject the counter intrinsic.
-                        bx.add_coverage_counter(self.instance, id, code_region);
+                        bx.add_coverage_counter(instance, id, code_region);
                     }
 
-                    let coverageinfo = bx.tcx().coverageinfo(self.instance.def_id());
+                    let coverageinfo = bx.tcx().coverageinfo(instance.def_id());
 
-                    let fn_name = bx.create_pgo_func_name_var(self.instance);
+                    let fn_name = bx.get_pgo_func_name_var(instance);
                     let hash = bx.const_u64(function_source_hash);
                     let num_counters = bx.const_u32(coverageinfo.num_counters);
-                    let index = bx.const_u32(u32::from(id));
+                    let index = bx.const_u32(id.zero_based_index());
                     debug!(
                         "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
                         fn_name, hash, num_counters, index,
@@ -34,11 +45,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
             CoverageKind::Expression { id, lhs, op, rhs } => {
-                bx.add_coverage_counter_expression(self.instance, id, lhs, op, rhs, code_region);
+                bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region);
             }
             CoverageKind::Unreachable => {
                 bx.add_coverage_unreachable(
-                    self.instance,
+                    instance,
                     code_region.expect("unreachable regions always have code regions"),
                 );
             }
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index ea59e183118..6bb20545f07 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -6,7 +6,7 @@ use rustc_middle::ty;
 use rustc_session::config::DebugInfo;
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{BytePos, Span};
-use rustc_target::abi::{LayoutOf, Size};
+use rustc_target::abi::Size;
 
 use super::operand::{OperandRef, OperandValue};
 use super::place::PlaceRef;
@@ -265,33 +265,25 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 None => continue,
             };
 
-            let mut layout = base.layout;
             let mut direct_offset = Size::ZERO;
             // FIXME(eddyb) use smallvec here.
             let mut indirect_offsets = vec![];
+            let mut place = base;
 
             for elem in &var.projection[..] {
                 match *elem {
                     mir::ProjectionElem::Deref => {
                         indirect_offsets.push(Size::ZERO);
-                        layout = bx.cx().layout_of(
-                            layout
-                                .ty
-                                .builtin_deref(true)
-                                .unwrap_or_else(|| {
-                                    span_bug!(var.source_info.span, "cannot deref `{}`", layout.ty)
-                                })
-                                .ty,
-                        );
+                        place = place.project_deref(bx);
                     }
                     mir::ProjectionElem::Field(field, _) => {
                         let i = field.index();
                         let offset = indirect_offsets.last_mut().unwrap_or(&mut direct_offset);
-                        *offset += layout.fields.offset(i);
-                        layout = layout.field(bx.cx(), i);
+                        *offset += place.layout.fields.offset(i);
+                        place = place.project_field(bx, i);
                     }
                     mir::ProjectionElem::Downcast(_, variant) => {
-                        layout = layout.for_variant(bx.cx(), variant);
+                        place = place.project_downcast(bx, variant);
                     }
                     _ => span_bug!(
                         var.source_info.span,
@@ -301,7 +293,39 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, direct_offset, &indirect_offsets);
+            // When targeting MSVC, create extra allocas for arguments instead of pointing multiple
+            // dbg_var_addr() calls into the same alloca with offsets. MSVC uses CodeView records
+            // not DWARF and LLVM doesn't support translating the resulting
+            // [DW_OP_deref, DW_OP_plus_uconst, offset, DW_OP_deref] debug info to CodeView.
+            // Creating extra allocas on the stack makes the resulting debug info simple enough
+            // that LLVM can generate correct CodeView records and thus the values appear in the
+            // debugger. (#83709)
+            let should_create_individual_allocas = bx.cx().sess().target.is_like_msvc
+                && self.mir.local_kind(local) == mir::LocalKind::Arg
+                // LLVM can handle simple things but anything more complex than just a direct
+                // offset or one indirect offset of 0 is too complex for it to generate CV records
+                // correctly.
+                && (direct_offset != Size::ZERO
+                    || !matches!(&indirect_offsets[..], [Size::ZERO] | []));
+
+            if should_create_individual_allocas {
+                // Create a variable which will be a pointer to the actual value
+                let ptr_ty = bx.tcx().mk_ty(ty::RawPtr(ty::TypeAndMut {
+                    mutbl: mir::Mutability::Mut,
+                    ty: place.layout.ty,
+                }));
+                let ptr_layout = bx.layout_of(ptr_ty);
+                let alloca = PlaceRef::alloca(bx, ptr_layout);
+                bx.set_var_name(alloca.llval, &(var.name.to_string() + ".dbg.spill"));
+
+                // Write the pointer to the variable
+                bx.store(place.llval, alloca.llval, alloca.align);
+
+                // Point the debug info to `*alloca` for the current variable
+                bx.dbg_var_addr(dbg_var, dbg_loc, alloca.llval, Size::ZERO, &[Size::ZERO]);
+            } else {
+                bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, direct_offset, &indirect_offsets);
+            }
         }
     }
 
@@ -372,7 +396,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         (var_ty, var_kind)
                     }
                     mir::VarDebugInfoContents::Const(c) => {
-                        let ty = self.monomorphize(c.literal.ty);
+                        let ty = self.monomorphize(c.ty());
                         (ty, VariableKind::LocalVariable)
                     }
                 };
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 80e3ed75b85..8502309b90e 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -125,19 +125,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let offset = args[1].immediate();
                 bx.gep(ptr, &[offset])
             }
-
-            sym::copy_nonoverlapping => {
-                copy_intrinsic(
-                    bx,
-                    false,
-                    false,
-                    substs.type_at(0),
-                    args[1].immediate(),
-                    args[0].immediate(),
-                    args[2].immediate(),
-                );
-                return;
-            }
             sym::copy => {
                 copy_intrinsic(
                     bx,
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index d31ececf130..91df67b53d2 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -149,8 +149,6 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         bx.set_personality_fn(cx.eh_personality());
     }
 
-    bx.sideeffect(false);
-
     let cleanup_kinds = analyze::cleanup_kinds(&mir);
     // Allocate a `Block` for every basic block, except
     // the start block, if nothing loops back to it.
@@ -284,9 +282,7 @@ fn create_funclets<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>>,
     IndexVec<mir::BasicBlock, Option<Bx::Funclet>>,
 ) {
-    block_bxs
-        .iter_enumerated()
-        .zip(cleanup_kinds)
+    iter::zip(block_bxs.iter_enumerated(), cleanup_kinds)
         .map(|((bb, &llbb), cleanup_kind)| {
             match *cleanup_kind {
                 CleanupKind::Funclet if base::wants_msvc_seh(bx.sess()) => {}
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 66d9d1a1e0c..a9e7ebf6d43 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -402,6 +402,18 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         downcast
     }
 
+    pub fn project_deref<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) -> Self {
+        let target_ty = self.layout.ty.builtin_deref(true).expect("failed to deref");
+        let layout = bx.layout_of(target_ty.ty);
+
+        PlaceRef {
+            llval: bx.load(self.llval, self.align),
+            llextra: None,
+            layout,
+            align: layout.align.abi,
+        }
+    }
+
     pub fn storage_live<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) {
         bx.lifetime_start(self.llval, self.layout.size);
     }
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index e3a6cabd600..9917c23f121 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -11,7 +11,7 @@ use rustc_apfloat::{ieee, Float, Round, Status};
 use rustc_hir::lang_items::LangItem;
 use rustc_middle::mir;
 use rustc_middle::ty::cast::{CastTy, IntTy};
-use rustc_middle::ty::layout::{HasTyCtxt, TyAndLayout};
+use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
 use rustc_span::source_map::{Span, DUMMY_SP};
 use rustc_span::symbol::sym;
@@ -325,7 +325,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
                                 let er = scalar.valid_range_exclusive(bx.cx());
                                 if er.end != er.start
-                                    && scalar.valid_range.end() > scalar.valid_range.start()
+                                    && scalar.valid_range.end() >= scalar.valid_range.start()
                                 {
                                     // We want `table[e as usize ± k]` to not
                                     // have bound checks, and this is the most
@@ -385,10 +385,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 bx.inttoptr(usize_llval, ll_t_out)
                             }
                             (CastTy::Float, CastTy::Int(IntTy::I)) => {
-                                cast_float_to_int(&mut bx, true, llval, ll_t_in, ll_t_out, cast)
+                                cast_float_to_int(&mut bx, true, llval, ll_t_in, ll_t_out)
                             }
                             (CastTy::Float, CastTy::Int(_)) => {
-                                cast_float_to_int(&mut bx, false, llval, ll_t_in, ll_t_out, cast)
+                                cast_float_to_int(&mut bx, false, llval, ll_t_in, ll_t_out)
                             }
                             _ => bug!("unsupported cast: {:?} to {:?}", operand.layout.ty, cast.ty),
                         };
@@ -424,7 +424,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 (bx, operand)
             }
 
-            mir::Rvalue::BinaryOp(op, ref lhs, ref rhs) => {
+            mir::Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => {
                 let lhs = self.codegen_operand(&mut bx, lhs);
                 let rhs = self.codegen_operand(&mut bx, rhs);
                 let llresult = match (lhs.val, rhs.val) {
@@ -453,7 +453,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 };
                 (bx, operand)
             }
-            mir::Rvalue::CheckedBinaryOp(op, ref lhs, ref rhs) => {
+            mir::Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => {
                 let lhs = self.codegen_operand(&mut bx, lhs);
                 let rhs = self.codegen_operand(&mut bx, rhs);
                 let result = self.codegen_scalar_checked_binop(
@@ -790,7 +790,6 @@ fn cast_float_to_int<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     x: Bx::Value,
     float_ty: Bx::Type,
     int_ty: Bx::Type,
-    int_layout: TyAndLayout<'tcx>,
 ) -> Bx::Value {
     if let Some(false) = bx.cx().sess().opts.debugging_opts.saturating_float_casts {
         return if signed { bx.fptosi(x, int_ty) } else { bx.fptoui(x, int_ty) };
@@ -891,134 +890,39 @@ fn cast_float_to_int<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let int_min = bx.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128);
     let zero = bx.cx().const_uint(int_ty, 0);
 
-    // The codegen here differs quite a bit depending on whether our builder's
-    // `fptosi` and `fptoui` instructions may trap for out-of-bounds values. If
-    // they don't trap then we can start doing everything inline with a
-    // `select` instruction because it's ok to execute `fptosi` and `fptoui`
-    // even if we don't use the results.
-    if !bx.fptosui_may_trap(x, int_ty) {
-        // Step 1 ...
-        let fptosui_result = if signed { bx.fptosi(x, int_ty) } else { bx.fptoui(x, int_ty) };
-        let less_or_nan = bx.fcmp(RealPredicate::RealULT, x, f_min);
-        let greater = bx.fcmp(RealPredicate::RealOGT, x, f_max);
-
-        // Step 2: We use two comparisons and two selects, with %s1 being the
-        // result:
-        //     %less_or_nan = fcmp ult %x, %f_min
-        //     %greater = fcmp olt %x, %f_max
-        //     %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result
-        //     %s1 = select %greater, int_ty::MAX, %s0
-        // Note that %less_or_nan uses an *unordered* comparison. This
-        // comparison is true if the operands are not comparable (i.e., if x is
-        // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if
-        // x is NaN.
-        //
-        // Performance note: Unordered comparison can be lowered to a "flipped"
-        // comparison and a negation, and the negation can be merged into the
-        // select. Therefore, it not necessarily any more expensive than a
-        // ordered ("normal") comparison. Whether these optimizations will be
-        // performed is ultimately up to the backend, but at least x86 does
-        // perform them.
-        let s0 = bx.select(less_or_nan, int_min, fptosui_result);
-        let s1 = bx.select(greater, int_max, s0);
-
-        // Step 3: NaN replacement.
-        // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN.
-        // Therefore we only need to execute this step for signed integer types.
-        if signed {
-            // LLVM has no isNaN predicate, so we use (x == x) instead
-            let cmp = bx.fcmp(RealPredicate::RealOEQ, x, x);
-            bx.select(cmp, s1, zero)
-        } else {
-            s1
-        }
+    // Step 1 ...
+    let fptosui_result = if signed { bx.fptosi(x, int_ty) } else { bx.fptoui(x, int_ty) };
+    let less_or_nan = bx.fcmp(RealPredicate::RealULT, x, f_min);
+    let greater = bx.fcmp(RealPredicate::RealOGT, x, f_max);
+
+    // Step 2: We use two comparisons and two selects, with %s1 being the
+    // result:
+    //     %less_or_nan = fcmp ult %x, %f_min
+    //     %greater = fcmp olt %x, %f_max
+    //     %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result
+    //     %s1 = select %greater, int_ty::MAX, %s0
+    // Note that %less_or_nan uses an *unordered* comparison. This
+    // comparison is true if the operands are not comparable (i.e., if x is
+    // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if
+    // x is NaN.
+    //
+    // Performance note: Unordered comparison can be lowered to a "flipped"
+    // comparison and a negation, and the negation can be merged into the
+    // select. Therefore, it not necessarily any more expensive than a
+    // ordered ("normal") comparison. Whether these optimizations will be
+    // performed is ultimately up to the backend, but at least x86 does
+    // perform them.
+    let s0 = bx.select(less_or_nan, int_min, fptosui_result);
+    let s1 = bx.select(greater, int_max, s0);
+
+    // Step 3: NaN replacement.
+    // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN.
+    // Therefore we only need to execute this step for signed integer types.
+    if signed {
+        // LLVM has no isNaN predicate, so we use (x == x) instead
+        let cmp = bx.fcmp(RealPredicate::RealOEQ, x, x);
+        bx.select(cmp, s1, zero)
     } else {
-        // In this case we cannot execute `fptosi` or `fptoui` and then later
-        // discard the result. The builder is telling us that these instructions
-        // will trap on out-of-bounds values, so we need to use basic blocks and
-        // control flow to avoid executing the `fptosi` and `fptoui`
-        // instructions.
-        //
-        // The general idea of what we're constructing here is, for f64 -> i32:
-        //
-        //      ;; block so far... %0 is the argument
-        //      %result = alloca i32, align 4
-        //      %inbound_lower = fcmp oge double %0, 0xC1E0000000000000
-        //      %inbound_upper = fcmp ole double %0, 0x41DFFFFFFFC00000
-        //      ;; match (inbound_lower, inbound_upper) {
-        //      ;;     (true, true) => %0 can be converted without trapping
-        //      ;;     (false, false) => %0 is a NaN
-        //      ;;     (true, false) => %0 is too large
-        //      ;;     (false, true) => %0 is too small
-        //      ;; }
-        //      ;;
-        //      ;; The (true, true) check, go to %convert if so.
-        //      %inbounds = and i1 %inbound_lower, %inbound_upper
-        //      br i1 %inbounds, label %convert, label %specialcase
-        //
-        //  convert:
-        //      %cvt = call i32 @llvm.wasm.trunc.signed.i32.f64(double %0)
-        //      store i32 %cvt, i32* %result, align 4
-        //      br label %done
-        //
-        //  specialcase:
-        //      ;; Handle the cases where the number is NaN, too large or too small
-        //
-        //      ;; Either (true, false) or (false, true)
-        //      %is_not_nan = or i1 %inbound_lower, %inbound_upper
-        //      ;; Figure out which saturated value we are interested in if not `NaN`
-        //      %saturated = select i1 %inbound_lower, i32 2147483647, i32 -2147483648
-        //      ;; Figure out between saturated and NaN representations
-        //      %result_nan = select i1 %is_not_nan, i32 %saturated, i32 0
-        //      store i32 %result_nan, i32* %result, align 4
-        //      br label %done
-        //
-        //  done:
-        //      %r = load i32, i32* %result, align 4
-        //      ;; ...
-        let done = bx.build_sibling_block("float_cast_done");
-        let mut convert = bx.build_sibling_block("float_cast_convert");
-        let mut specialcase = bx.build_sibling_block("float_cast_specialcase");
-
-        let result = PlaceRef::alloca(bx, int_layout);
-        result.storage_live(bx);
-
-        // Use control flow to figure out whether we can execute `fptosi` in a
-        // basic block, or whether we go to a different basic block to implement
-        // the saturating logic.
-        let inbound_lower = bx.fcmp(RealPredicate::RealOGE, x, f_min);
-        let inbound_upper = bx.fcmp(RealPredicate::RealOLE, x, f_max);
-        let inbounds = bx.and(inbound_lower, inbound_upper);
-        bx.cond_br(inbounds, convert.llbb(), specialcase.llbb());
-
-        // Translation of the `convert` basic block
-        let cvt = if signed { convert.fptosi(x, int_ty) } else { convert.fptoui(x, int_ty) };
-        convert.store(cvt, result.llval, result.align);
-        convert.br(done.llbb());
-
-        // Translation of the `specialcase` basic block. Note that like above
-        // we try to be a bit clever here for unsigned conversions. In those
-        // cases the `int_min` is zero so we don't need two select instructions,
-        // just one to choose whether we need `int_max` or not. If
-        // `inbound_lower` is true then we're guaranteed to not be `NaN` and
-        // since we're greater than zero we must be saturating to `int_max`. If
-        // `inbound_lower` is false then we're either NaN or less than zero, so
-        // we saturate to zero.
-        let result_nan = if signed {
-            let is_not_nan = specialcase.or(inbound_lower, inbound_upper);
-            let saturated = specialcase.select(inbound_lower, int_max, int_min);
-            specialcase.select(is_not_nan, saturated, zero)
-        } else {
-            specialcase.select(inbound_lower, int_max, int_min)
-        };
-        specialcase.store(result_nan, result.llval, result.align);
-        specialcase.br(done.llbb());
-
-        // Translation of the `done` basic block, positioning ourselves to
-        // continue from that point as well.
-        *bx = done;
-        let ret = bx.load(result.llval, result.align);
-        result.storage_dead(bx);
-        ret
+        s1
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index 6f74ba77d4c..fe7f6288adb 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -112,7 +112,27 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 bx
             }
             mir::StatementKind::Coverage(box ref coverage) => {
-                self.codegen_coverage(&mut bx, coverage.clone());
+                self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
+                bx
+            }
+            mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
+                ref src,
+                ref dst,
+                ref count,
+            }) => {
+                let dst_val = self.codegen_operand(&mut bx, dst);
+                let src_val = self.codegen_operand(&mut bx, src);
+                let count = self.codegen_operand(&mut bx, count).immediate();
+                let pointee_layout = dst_val
+                    .layout
+                    .pointee_info_at(&mut bx, rustc_target::abi::Size::ZERO)
+                    .expect("Expected pointer");
+                let bytes = bx.mul(count, bx.const_usize(pointee_layout.size.bytes()));
+
+                let align = pointee_layout.align;
+                let dst = dst_val.immediate();
+                let src = src_val.immediate();
+                bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
                 bx
             }
             mir::StatementKind::FakeRead(..)
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index fd18f42f2dd..4e987908b4e 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -26,6 +26,7 @@ const ARM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("vfp2", Some(sym::arm_target_feature)),
     ("vfp3", Some(sym::arm_target_feature)),
     ("vfp4", Some(sym::arm_target_feature)),
+    ("fp-armv8", Some(sym::arm_target_feature)),
     // This is needed for inline assembly, but shouldn't be stabilized as-is
     // since it should be enabled per-function using #[instruction_set], not
     // #[target_feature].
@@ -160,7 +161,7 @@ pub fn supported_target_features(sess: &Session) -> &'static [(&'static str, Opt
         "mips" | "mips64" => MIPS_ALLOWED_FEATURES,
         "powerpc" | "powerpc64" => POWERPC_ALLOWED_FEATURES,
         "riscv32" | "riscv64" => RISCV_ALLOWED_FEATURES,
-        "wasm32" => WASM_ALLOWED_FEATURES,
+        "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES,
         _ => &[],
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index d5bd2780388..1bc05f30e5c 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -171,7 +171,6 @@ pub trait BuilderMethods<'a, 'tcx>:
     fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
     fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
-    fn fptosui_may_trap(&self, val: Self::Value, dest_ty: Self::Type) -> bool;
     fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
diff --git a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs
index 95bddfb4b41..cbf570dba4c 100644
--- a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs
@@ -1,14 +1,26 @@
 use super::BackendTypes;
+use rustc_hir::def_id::DefId;
 use rustc_middle::mir::coverage::*;
 use rustc_middle::ty::Instance;
 
-pub trait CoverageInfoMethods: BackendTypes {
+pub trait CoverageInfoMethods<'tcx>: BackendTypes {
     fn coverageinfo_finalize(&self);
+
+    /// Codegen a small function that will never be called, with one counter
+    /// that will never be incremented, that gives LLVM coverage tools a
+    /// function definition it needs in order to resolve coverage map references
+    /// to unused functions. This is necessary so unused functions will appear
+    /// as uncovered (coverage execution count `0`) in LLVM coverage reports.
+    fn define_unused_fn(&self, def_id: DefId);
+
+    /// For LLVM codegen, returns a function-specific `Value` for a global
+    /// string, to hold the function name passed to LLVM intrinsic
+    /// `instrprof.increment()`. The `Value` is only created once per instance.
+    /// Multiple invocations with the same instance return the same `Value`.
+    fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value;
 }
 
 pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes {
-    fn create_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value;
-
     /// Returns true if the function source hash was added to the coverage map (even if it had
     /// already been added, for this instance). Returns false *only* if `-Z instrument-coverage` is
     /// not enabled (a coverage map is not being generated).
diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
index ac3c99f9c90..777436ad2ae 100644
--- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
@@ -20,9 +20,10 @@ pub trait IntrinsicCallMethods<'tcx>: BackendTypes {
     fn abort(&mut self);
     fn assume(&mut self, val: Self::Value);
     fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value;
-    /// Normally, sideeffect is only emitted if -Zinsert-sideeffect is passed;
-    /// in some cases though we want to emit it regardless.
-    fn sideeffect(&mut self, unconditional: bool);
+    /// Emits a forced side effect.
+    ///
+    /// Currently has any effect only when LLVM versions prior to 12.0 are used as the backend.
+    fn sideeffect(&mut self);
     /// Trait method used to inject `va_start` on the "spoofed" `VaListImpl` in
     /// Rust defined C-variadic functions.
     fn va_start(&mut self, val: Self::Value) -> Self::Value;
diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs
index 8ada6c10479..be2e0ea230f 100644
--- a/compiler/rustc_codegen_ssa/src/traits/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs
@@ -58,7 +58,7 @@ pub trait CodegenMethods<'tcx>:
     + MiscMethods<'tcx>
     + ConstMethods<'tcx>
     + StaticMethods
-    + CoverageInfoMethods
+    + CoverageInfoMethods<'tcx>
     + DebugInfoMethods<'tcx>
     + AsmMethods
     + PreDefineMethods<'tcx>
@@ -74,7 +74,7 @@ impl<'tcx, T> CodegenMethods<'tcx> for T where
         + MiscMethods<'tcx>
         + ConstMethods<'tcx>
         + StaticMethods
-        + CoverageInfoMethods
+        + CoverageInfoMethods<'tcx>
         + DebugInfoMethods<'tcx>
         + AsmMethods
         + PreDefineMethods<'tcx>
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index 818c4364256..d0a5fe1f2a6 100644
--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2018"
 doctest = false
 
 [dependencies]
-arrayvec = { version = "0.5.1", default-features = false }
+arrayvec = { version = "0.7", default-features = false }
 ena = "0.14"
 indexmap = "1.5.1"
 tracing = "0.1"
@@ -19,8 +19,8 @@ rustc_graphviz = { path = "../rustc_graphviz" }
 cfg-if = "0.1.2"
 crossbeam-utils = { version = "0.7", features = ["nightly"] }
 stable_deref_trait = "1.0.0"
-rayon = { version = "0.3.0", package = "rustc-rayon" }
-rayon-core = { version = "0.3.0", package = "rustc-rayon-core" }
+rayon = { version = "0.3.1", package = "rustc-rayon" }
+rayon-core = { version = "0.3.1", package = "rustc-rayon-core" }
 rustc-hash = "1.1.0"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_index = { path = "../rustc_index", package = "rustc_index" }
@@ -36,3 +36,6 @@ features = ["nightly"]
 
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["fileapi", "psapi"] }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+memmap2 = "0.2.1"
diff --git a/compiler/rustc_data_structures/src/fingerprint.rs b/compiler/rustc_data_structures/src/fingerprint.rs
index 08c3419a842..c0c0e7be3ca 100644
--- a/compiler/rustc_data_structures/src/fingerprint.rs
+++ b/compiler/rustc_data_structures/src/fingerprint.rs
@@ -1,25 +1,33 @@
 use crate::stable_hasher;
-use rustc_serialize::{
-    opaque::{self, EncodeResult, FileEncodeResult},
-    Decodable, Encodable,
-};
+use rustc_serialize::{Decodable, Encodable};
+use std::convert::TryInto;
 use std::hash::{Hash, Hasher};
-use std::mem::{self, MaybeUninit};
 
 #[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Copy)]
+#[repr(C)]
 pub struct Fingerprint(u64, u64);
 
 impl Fingerprint {
     pub const ZERO: Fingerprint = Fingerprint(0, 0);
 
     #[inline]
+    pub fn new(_0: u64, _1: u64) -> Fingerprint {
+        Fingerprint(_0, _1)
+    }
+
+    #[inline]
     pub fn from_smaller_hash(hash: u64) -> Fingerprint {
         Fingerprint(hash, hash)
     }
 
     #[inline]
     pub fn to_smaller_hash(&self) -> u64 {
-        self.0
+        // Even though both halves of the fingerprint are expected to be good
+        // quality hash values, let's still combine the two values because the
+        // Fingerprints in DefPathHash have the StableCrateId portion which is
+        // the same for all DefPathHashes from the same crate. Combining the
+        // two halfs makes sure we get a good quality hash in such cases too.
+        self.0.wrapping_mul(3).wrapping_add(self.1)
     }
 
     #[inline]
@@ -53,14 +61,27 @@ impl Fingerprint {
         format!("{:x}{:x}", self.0, self.1)
     }
 
-    pub fn decode_opaque(decoder: &mut opaque::Decoder<'_>) -> Result<Fingerprint, String> {
-        let mut bytes: [MaybeUninit<u8>; 16] = MaybeUninit::uninit_array();
+    #[inline]
+    pub fn to_le_bytes(&self) -> [u8; 16] {
+        // This seems to optimize to the same machine code as
+        // `unsafe { mem::transmute(*k) }`. Well done, LLVM! :)
+        let mut result = [0u8; 16];
 
-        decoder.read_raw_bytes(&mut bytes)?;
+        let first_half: &mut [u8; 8] = (&mut result[0..8]).try_into().unwrap();
+        *first_half = self.0.to_le_bytes();
 
-        let [l, r]: [u64; 2] = unsafe { mem::transmute(bytes) };
+        let second_half: &mut [u8; 8] = (&mut result[8..16]).try_into().unwrap();
+        *second_half = self.1.to_le_bytes();
 
-        Ok(Fingerprint(u64::from_le(l), u64::from_le(r)))
+        result
+    }
+
+    #[inline]
+    pub fn from_le_bytes(bytes: [u8; 16]) -> Fingerprint {
+        Fingerprint(
+            u64::from_le_bytes(bytes[0..8].try_into().unwrap()),
+            u64::from_le_bytes(bytes[8..16].try_into().unwrap()),
+        )
     }
 }
 
@@ -92,8 +113,19 @@ impl<H: Hasher> FingerprintHasher for H {
 impl FingerprintHasher for crate::unhash::Unhasher {
     #[inline]
     fn write_fingerprint(&mut self, fingerprint: &Fingerprint) {
-        // `Unhasher` only wants a single `u64`
-        self.write_u64(fingerprint.0);
+        // Even though both halves of the fingerprint are expected to be good
+        // quality hash values, let's still combine the two values because the
+        // Fingerprints in DefPathHash have the StableCrateId portion which is
+        // the same for all DefPathHashes from the same crate. Combining the
+        // two halfs makes sure we get a good quality hash in such cases too.
+        //
+        // Since `Unhasher` is used only in the context of HashMaps, it is OK
+        // to combine the two components in an order-independent way (which is
+        // cheaper than the more robust Fingerprint::to_smaller_hash()). For
+        // HashMaps we don't really care if Fingerprint(x,y) and
+        // Fingerprint(y, x) result in the same hash value. Collision
+        // probability will still be much better than with FxHash.
+        self.write_u64(fingerprint.0.wrapping_add(fingerprint.1));
     }
 }
 
@@ -108,55 +140,19 @@ impl stable_hasher::StableHasherResult for Fingerprint {
 impl_stable_hash_via_hash!(Fingerprint);
 
 impl<E: rustc_serialize::Encoder> Encodable<E> for Fingerprint {
+    #[inline]
     fn encode(&self, s: &mut E) -> Result<(), E::Error> {
-        s.encode_fingerprint(self)
+        s.emit_raw_bytes(&self.to_le_bytes()[..])?;
+        Ok(())
     }
 }
 
 impl<D: rustc_serialize::Decoder> Decodable<D> for Fingerprint {
+    #[inline]
     fn decode(d: &mut D) -> Result<Self, D::Error> {
-        d.decode_fingerprint()
-    }
-}
-
-pub trait FingerprintEncoder: rustc_serialize::Encoder {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> Result<(), Self::Error>;
-}
-
-pub trait FingerprintDecoder: rustc_serialize::Decoder {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, Self::Error>;
-}
-
-impl<E: rustc_serialize::Encoder> FingerprintEncoder for E {
-    default fn encode_fingerprint(&mut self, _: &Fingerprint) -> Result<(), E::Error> {
-        panic!("Cannot encode `Fingerprint` with `{}`", std::any::type_name::<E>());
-    }
-}
-
-impl FingerprintEncoder for opaque::Encoder {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> EncodeResult {
-        let bytes: [u8; 16] = unsafe { mem::transmute([f.0.to_le(), f.1.to_le()]) };
-        self.emit_raw_bytes(&bytes);
-        Ok(())
-    }
-}
-
-impl FingerprintEncoder for opaque::FileEncoder {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> FileEncodeResult {
-        let bytes: [u8; 16] = unsafe { mem::transmute([f.0.to_le(), f.1.to_le()]) };
-        self.emit_raw_bytes(&bytes)
-    }
-}
-
-impl<D: rustc_serialize::Decoder> FingerprintDecoder for D {
-    default fn decode_fingerprint(&mut self) -> Result<Fingerprint, D::Error> {
-        panic!("Cannot decode `Fingerprint` with `{}`", std::any::type_name::<D>());
-    }
-}
-
-impl FingerprintDecoder for opaque::Decoder<'_> {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, String> {
-        Fingerprint::decode_opaque(self)
+        let mut bytes = [0u8; 16];
+        d.read_raw_bytes_into(&mut bytes[..])?;
+        Ok(Fingerprint::from_le_bytes(bytes))
     }
 }
 
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index fcb2bca7b4c..adbb98fa750 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -13,7 +13,6 @@
 #![feature(unboxed_closures)]
 #![feature(generator_trait)]
 #![feature(fn_traits)]
-#![feature(int_bits_const)]
 #![feature(min_specialization)]
 #![feature(auto_traits)]
 #![feature(nll)]
@@ -85,6 +84,7 @@ pub mod snapshot_map;
 pub mod stable_map;
 pub mod svh;
 pub use ena::snapshot_vec;
+pub mod memmap;
 pub mod sorted_map;
 pub mod stable_set;
 #[macro_use]
diff --git a/compiler/rustc_data_structures/src/macros.rs b/compiler/rustc_data_structures/src/macros.rs
index b918ed9458c..48dfbba7504 100644
--- a/compiler/rustc_data_structures/src/macros.rs
+++ b/compiler/rustc_data_structures/src/macros.rs
@@ -9,11 +9,11 @@ macro_rules! static_assert_size {
 #[macro_export]
 macro_rules! enum_from_u32 {
     ($(#[$attr:meta])* pub enum $name:ident {
-        $($variant:ident = $e:expr,)*
+        $($(#[$var_attr:meta])* $variant:ident = $e:expr,)*
     }) => {
         $(#[$attr])*
         pub enum $name {
-            $($variant = $e),*
+            $($(#[$var_attr])* $variant = $e),*
         }
 
         impl $name {
@@ -26,11 +26,11 @@ macro_rules! enum_from_u32 {
         }
     };
     ($(#[$attr:meta])* pub enum $name:ident {
-        $($variant:ident,)*
+        $($(#[$var_attr:meta])* $variant:ident,)*
     }) => {
         $(#[$attr])*
         pub enum $name {
-            $($variant,)*
+            $($(#[$var_attr])* $variant,)*
         }
 
         impl $name {
diff --git a/compiler/rustc_data_structures/src/memmap.rs b/compiler/rustc_data_structures/src/memmap.rs
new file mode 100644
index 00000000000..26b26415eea
--- /dev/null
+++ b/compiler/rustc_data_structures/src/memmap.rs
@@ -0,0 +1,47 @@
+use std::fs::File;
+use std::io;
+use std::ops::Deref;
+
+use crate::owning_ref::StableAddress;
+
+/// A trivial wrapper for [`memmap2::Mmap`] that implements [`StableAddress`].
+#[cfg(not(target_arch = "wasm32"))]
+pub struct Mmap(memmap2::Mmap);
+
+#[cfg(target_arch = "wasm32")]
+pub struct Mmap(Vec<u8>);
+
+#[cfg(not(target_arch = "wasm32"))]
+impl Mmap {
+    #[inline]
+    pub unsafe fn map(file: File) -> io::Result<Self> {
+        memmap2::Mmap::map(&file).map(Mmap)
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+impl Mmap {
+    #[inline]
+    pub unsafe fn map(mut file: File) -> io::Result<Self> {
+        use std::io::Read;
+
+        let mut data = Vec::new();
+        file.read_to_end(&mut data)?;
+        Ok(Mmap(data))
+    }
+}
+
+impl Deref for Mmap {
+    type Target = [u8];
+
+    #[inline]
+    fn deref(&self) -> &[u8] {
+        &*self.0
+    }
+}
+
+// SAFETY: On architectures other than WASM, mmap is used as backing storage. The address of this
+// memory map is stable. On WASM, `Vec<u8>` is used as backing storage. The `Mmap` type doesn't
+// export any function that can cause the `Vec` to be re-allocated. As such the address of the
+// bytes inside this `Vec` is stable.
+unsafe impl StableAddress for Mmap {}
diff --git a/compiler/rustc_data_structures/src/sso/map.rs b/compiler/rustc_data_structures/src/sso/map.rs
index 06e8442d475..e249886e9bc 100644
--- a/compiler/rustc_data_structures/src/sso/map.rs
+++ b/compiler/rustc_data_structures/src/sso/map.rs
@@ -18,11 +18,8 @@ use std::ops::Index;
 // for reasonably small arrays that stay
 // small in vast majority of cases.
 //
-// '8' is choosen as a sane default, to be
+// '8' is chosen as a sane default, to be
 // reevaluated later.
-//
-// Note: As of now ArrayVec design prevents
-//       us from making it user-customizable.
 const SSO_ARRAY_SIZE: usize = 8;
 
 /// Small-storage-optimized implementation of a map.
@@ -70,7 +67,7 @@ const SSO_ARRAY_SIZE: usize = 8;
 
 #[derive(Clone)]
 pub enum SsoHashMap<K, V> {
-    Array(ArrayVec<[(K, V); SSO_ARRAY_SIZE]>),
+    Array(ArrayVec<(K, V), SSO_ARRAY_SIZE>),
     Map(FxHashMap<K, V>),
 }
 
@@ -411,7 +408,7 @@ where
 
 impl<K, V> IntoIterator for SsoHashMap<K, V> {
     type IntoIter = EitherIter<
-        <ArrayVec<[(K, V); 8]> as IntoIterator>::IntoIter,
+        <ArrayVec<(K, V), 8> as IntoIterator>::IntoIter,
         <FxHashMap<K, V> as IntoIterator>::IntoIter,
     >;
     type Item = <Self::IntoIter as Iterator>::Item;
@@ -441,7 +438,7 @@ fn adapt_array_mut_it<K, V>(pair: &'a mut (K, V)) -> (&'a K, &'a mut V) {
 impl<'a, K, V> IntoIterator for &'a SsoHashMap<K, V> {
     type IntoIter = EitherIter<
         std::iter::Map<
-            <&'a ArrayVec<[(K, V); 8]> as IntoIterator>::IntoIter,
+            <&'a ArrayVec<(K, V), 8> as IntoIterator>::IntoIter,
             fn(&'a (K, V)) -> (&'a K, &'a V),
         >,
         <&'a FxHashMap<K, V> as IntoIterator>::IntoIter,
@@ -459,7 +456,7 @@ impl<'a, K, V> IntoIterator for &'a SsoHashMap<K, V> {
 impl<'a, K, V> IntoIterator for &'a mut SsoHashMap<K, V> {
     type IntoIter = EitherIter<
         std::iter::Map<
-            <&'a mut ArrayVec<[(K, V); 8]> as IntoIterator>::IntoIter,
+            <&'a mut ArrayVec<(K, V), 8> as IntoIterator>::IntoIter,
             fn(&'a mut (K, V)) -> (&'a K, &'a mut V),
         >,
         <&'a mut FxHashMap<K, V> as IntoIterator>::IntoIter,
diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs
index 3850c9b74fd..ff28784a1dc 100644
--- a/compiler/rustc_data_structures/src/stable_hasher.rs
+++ b/compiler/rustc_data_structures/src/stable_hasher.rs
@@ -35,6 +35,7 @@ impl StableHasher {
         StableHasher { state: SipHasher128::new_with_keys(0, 0) }
     }
 
+    #[inline]
     pub fn finish<W: StableHasherResult>(self) -> W {
         W::finish(self)
     }
diff --git a/compiler/rustc_data_structures/src/svh.rs b/compiler/rustc_data_structures/src/svh.rs
index 02103de2e8d..ce90fbacaa4 100644
--- a/compiler/rustc_data_structures/src/svh.rs
+++ b/compiler/rustc_data_structures/src/svh.rs
@@ -19,7 +19,7 @@ pub struct Svh {
 impl Svh {
     /// Creates a new `Svh` given the hash. If you actually want to
     /// compute the SVH from some HIR, you want the `calculate_svh`
-    /// function found in `librustc_incremental`.
+    /// function found in `rustc_incremental`.
     pub fn new(hash: u64) -> Svh {
         Svh { hash }
     }
diff --git a/compiler/rustc_data_structures/src/tagged_ptr/drop.rs b/compiler/rustc_data_structures/src/tagged_ptr/drop.rs
index 63f64beae5a..d44ccd368b3 100644
--- a/compiler/rustc_data_structures/src/tagged_ptr/drop.rs
+++ b/compiler/rustc_data_structures/src/tagged_ptr/drop.rs
@@ -42,18 +42,9 @@ where
     pub fn pointer_ref(&self) -> &P::Target {
         self.raw.pointer_ref()
     }
-    pub fn pointer_mut(&mut self) -> &mut P::Target
-    where
-        P: std::ops::DerefMut,
-    {
-        self.raw.pointer_mut()
-    }
     pub fn tag(&self) -> T {
         self.raw.tag()
     }
-    pub fn set_tag(&mut self, tag: T) {
-        self.raw.set_tag(tag);
-    }
 }
 
 impl<P, T, const COMPARE_PACKED: bool> std::ops::Deref for TaggedPtr<P, T, COMPARE_PACKED>
diff --git a/compiler/rustc_data_structures/src/thin_vec.rs b/compiler/rustc_data_structures/src/thin_vec.rs
index 4d673fd5cf9..00e30473498 100644
--- a/compiler/rustc_data_structures/src/thin_vec.rs
+++ b/compiler/rustc_data_structures/src/thin_vec.rs
@@ -1,5 +1,7 @@
 use crate::stable_hasher::{HashStable, StableHasher};
 
+use std::iter::FromIterator;
+
 /// A vector type optimized for cases where this size is usually 0 (cf. `SmallVector`).
 /// The `Option<Box<..>>` wrapping allows us to represent a zero sized vector with `None`,
 /// which uses only a single (null) pointer.
@@ -10,6 +12,14 @@ impl<T> ThinVec<T> {
     pub fn new() -> Self {
         ThinVec(None)
     }
+
+    pub fn iter(&self) -> std::slice::Iter<'_, T> {
+        self.into_iter()
+    }
+
+    pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> {
+        self.into_iter()
+    }
 }
 
 impl<T> From<Vec<T>> for ThinVec<T> {
@@ -46,6 +56,42 @@ impl<T> ::std::ops::DerefMut for ThinVec<T> {
     }
 }
 
+impl<T> FromIterator<T> for ThinVec<T> {
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
+        // `Vec::from_iter()` should not allocate if the iterator is empty.
+        let vec: Vec<_> = iter.into_iter().collect();
+        if vec.is_empty() { ThinVec(None) } else { ThinVec(Some(Box::new(vec))) }
+    }
+}
+
+impl<T> IntoIterator for ThinVec<T> {
+    type Item = T;
+    type IntoIter = std::vec::IntoIter<T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        // This is still performant because `Vec::new()` does not allocate.
+        self.0.map_or_else(Vec::new, |ptr| *ptr).into_iter()
+    }
+}
+
+impl<'a, T> IntoIterator for &'a ThinVec<T> {
+    type Item = &'a T;
+    type IntoIter = std::slice::Iter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.as_ref().iter()
+    }
+}
+
+impl<'a, T> IntoIterator for &'a mut ThinVec<T> {
+    type Item = &'a mut T;
+    type IntoIter = std::slice::IterMut<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.as_mut().iter_mut()
+    }
+}
+
 impl<T> Extend<T> for ThinVec<T> {
     fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
         match *self {
@@ -80,3 +126,6 @@ impl<T> Default for ThinVec<T> {
         Self(None)
     }
 }
+
+#[cfg(test)]
+mod tests;
diff --git a/compiler/rustc_data_structures/src/thin_vec/tests.rs b/compiler/rustc_data_structures/src/thin_vec/tests.rs
new file mode 100644
index 00000000000..5abfd939373
--- /dev/null
+++ b/compiler/rustc_data_structures/src/thin_vec/tests.rs
@@ -0,0 +1,42 @@
+use super::*;
+
+impl<T> ThinVec<T> {
+    fn into_vec(self) -> Vec<T> {
+        self.into()
+    }
+}
+
+#[test]
+fn test_from_iterator() {
+    assert_eq!(std::iter::empty().collect::<ThinVec<String>>().into_vec(), Vec::<String>::new());
+    assert_eq!(std::iter::once(42).collect::<ThinVec<_>>().into_vec(), vec![42]);
+    assert_eq!(vec![1, 2].into_iter().collect::<ThinVec<_>>().into_vec(), vec![1, 2]);
+    assert_eq!(vec![1, 2, 3].into_iter().collect::<ThinVec<_>>().into_vec(), vec![1, 2, 3]);
+}
+
+#[test]
+fn test_into_iterator_owned() {
+    assert_eq!(ThinVec::new().into_iter().collect::<Vec<String>>(), Vec::<String>::new());
+    assert_eq!(ThinVec::from(vec![1]).into_iter().collect::<Vec<_>>(), vec![1]);
+    assert_eq!(ThinVec::from(vec![1, 2]).into_iter().collect::<Vec<_>>(), vec![1, 2]);
+    assert_eq!(ThinVec::from(vec![1, 2, 3]).into_iter().collect::<Vec<_>>(), vec![1, 2, 3]);
+}
+
+#[test]
+fn test_into_iterator_ref() {
+    assert_eq!(ThinVec::new().iter().collect::<Vec<&String>>(), Vec::<&String>::new());
+    assert_eq!(ThinVec::from(vec![1]).iter().collect::<Vec<_>>(), vec![&1]);
+    assert_eq!(ThinVec::from(vec![1, 2]).iter().collect::<Vec<_>>(), vec![&1, &2]);
+    assert_eq!(ThinVec::from(vec![1, 2, 3]).iter().collect::<Vec<_>>(), vec![&1, &2, &3]);
+}
+
+#[test]
+fn test_into_iterator_ref_mut() {
+    assert_eq!(ThinVec::new().iter_mut().collect::<Vec<&mut String>>(), Vec::<&mut String>::new());
+    assert_eq!(ThinVec::from(vec![1]).iter_mut().collect::<Vec<_>>(), vec![&mut 1]);
+    assert_eq!(ThinVec::from(vec![1, 2]).iter_mut().collect::<Vec<_>>(), vec![&mut 1, &mut 2]);
+    assert_eq!(
+        ThinVec::from(vec![1, 2, 3]).iter_mut().collect::<Vec<_>>(),
+        vec![&mut 1, &mut 2, &mut 3],
+    );
+}
diff --git a/compiler/rustc_data_structures/src/tiny_list.rs b/compiler/rustc_data_structures/src/tiny_list.rs
index e94a0c6eb59..f88bcc29481 100644
--- a/compiler/rustc_data_structures/src/tiny_list.rs
+++ b/compiler/rustc_data_structures/src/tiny_list.rs
@@ -15,7 +15,7 @@
 mod tests;
 
 #[derive(Clone)]
-pub struct TinyList<T: PartialEq> {
+pub struct TinyList<T> {
     head: Option<Element<T>>,
 }
 
@@ -56,20 +56,10 @@ impl<T: PartialEq> TinyList<T> {
         }
         false
     }
-
-    #[inline]
-    pub fn len(&self) -> usize {
-        let (mut elem, mut count) = (self.head.as_ref(), 0);
-        while let Some(ref e) = elem {
-            count += 1;
-            elem = e.next.as_deref();
-        }
-        count
-    }
 }
 
 #[derive(Clone)]
-struct Element<T: PartialEq> {
+struct Element<T> {
     data: T,
     next: Option<Box<Element<T>>>,
 }
diff --git a/compiler/rustc_data_structures/src/tiny_list/tests.rs b/compiler/rustc_data_structures/src/tiny_list/tests.rs
index a8ae2bc8727..c0334d2e23e 100644
--- a/compiler/rustc_data_structures/src/tiny_list/tests.rs
+++ b/compiler/rustc_data_structures/src/tiny_list/tests.rs
@@ -3,6 +3,17 @@ use super::*;
 extern crate test;
 use test::{black_box, Bencher};
 
+impl<T> TinyList<T> {
+    fn len(&self) -> usize {
+        let (mut elem, mut count) = (self.head.as_ref(), 0);
+        while let Some(ref e) = elem {
+            count += 1;
+            elem = e.next.as_deref();
+        }
+        count
+    }
+}
+
 #[test]
 fn test_contains_and_insert() {
     fn do_insert(i: u32) -> bool {
diff --git a/compiler/rustc_data_structures/src/transitive_relation.rs b/compiler/rustc_data_structures/src/transitive_relation.rs
index 2e1512b3929..ccf8bd69ebd 100644
--- a/compiler/rustc_data_structures/src/transitive_relation.rs
+++ b/compiler/rustc_data_structures/src/transitive_relation.rs
@@ -9,7 +9,7 @@ use std::mem;
 mod tests;
 
 #[derive(Clone, Debug)]
-pub struct TransitiveRelation<T: Eq + Hash> {
+pub struct TransitiveRelation<T> {
     // List of elements. This is used to map from a T to a usize.
     elements: FxIndexSet<T>,
 
@@ -49,7 +49,7 @@ struct Edge {
     target: Index,
 }
 
-impl<T: Clone + Debug + Eq + Hash> TransitiveRelation<T> {
+impl<T: Eq + Hash> TransitiveRelation<T> {
     pub fn is_empty(&self) -> bool {
         self.edges.is_empty()
     }
@@ -322,12 +322,6 @@ impl<T: Clone + Debug + Eq + Hash> TransitiveRelation<T> {
             .collect()
     }
 
-    /// A "best" parent in some sense. See `parents` and
-    /// `postdom_upper_bound` for more details.
-    pub fn postdom_parent(&self, a: &T) -> Option<&T> {
-        self.mutual_immediate_postdominator(self.parents(a))
-    }
-
     fn with_closure<OP, R>(&self, op: OP) -> R
     where
         OP: FnOnce(&BitMatrix<usize, usize>) -> R,
diff --git a/compiler/rustc_data_structures/src/transitive_relation/tests.rs b/compiler/rustc_data_structures/src/transitive_relation/tests.rs
index ca90ba176ae..9fa7224376c 100644
--- a/compiler/rustc_data_structures/src/transitive_relation/tests.rs
+++ b/compiler/rustc_data_structures/src/transitive_relation/tests.rs
@@ -1,5 +1,13 @@
 use super::*;
 
+impl<T: Eq + Hash> TransitiveRelation<T> {
+    /// A "best" parent in some sense. See `parents` and
+    /// `postdom_upper_bound` for more details.
+    fn postdom_parent(&self, a: &T) -> Option<&T> {
+        self.mutual_immediate_postdominator(self.parents(a))
+    }
+}
+
 #[test]
 fn test_one_step() {
     let mut relation = TransitiveRelation::default();
diff --git a/compiler/rustc_data_structures/src/work_queue.rs b/compiler/rustc_data_structures/src/work_queue.rs
index cc562bc1e4d..10317f1afff 100644
--- a/compiler/rustc_data_structures/src/work_queue.rs
+++ b/compiler/rustc_data_structures/src/work_queue.rs
@@ -41,10 +41,4 @@ impl<T: Idx> WorkQueue<T> {
             None
         }
     }
-
-    /// Returns `true` if nothing is enqueued.
-    #[inline]
-    pub fn is_empty(&self) -> bool {
-        self.deque.is_empty()
-    }
 }
diff --git a/compiler/rustc_driver/Cargo.toml b/compiler/rustc_driver/Cargo.toml
index 47cff34cd3e..c521f2041d8 100644
--- a/compiler/rustc_driver/Cargo.toml
+++ b/compiler/rustc_driver/Cargo.toml
@@ -12,7 +12,7 @@ libc = "0.2"
 atty = "0.2"
 tracing = { version = "0.1.25" }
 tracing-subscriber = { version = "0.2.16", default-features = false, features = ["fmt", "env-filter", "smallvec", "parking_lot", "ansi"] }
-tracing-tree = "0.1.8"
+tracing-tree = "0.1.9"
 rustc_middle = { path = "../rustc_middle" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_target = { path = "../rustc_target" }
@@ -34,6 +34,8 @@ rustc_interface = { path = "../rustc_interface" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_span = { path = "../rustc_span" }
+rustc_mir_build = { path = "../rustc_mir_build" }
+rustc_typeck = { path = "../rustc_typeck" }
 
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["consoleapi", "debugapi", "processenv"] }
diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs
index 79bb21b29fc..51699403a37 100644
--- a/compiler/rustc_driver/src/lib.rs
+++ b/compiler/rustc_driver/src/lib.rs
@@ -146,6 +146,7 @@ impl<'a, 'b> RunCompiler<'a, 'b> {
     pub fn new(at_args: &'a [String], callbacks: &'b mut (dyn Callbacks + Send)) -> Self {
         Self { at_args, callbacks, file_loader: None, emitter: None, make_codegen_backend: None }
     }
+    /// Used by cg_clif.
     pub fn set_make_codegen_backend(
         &mut self,
         make_codegen_backend: Option<
@@ -155,10 +156,12 @@ impl<'a, 'b> RunCompiler<'a, 'b> {
         self.make_codegen_backend = make_codegen_backend;
         self
     }
+    /// Used by RLS.
     pub fn set_emitter(&mut self, emitter: Option<Box<dyn Write + Send>>) -> &mut Self {
         self.emitter = emitter;
         self
     }
+    /// Used by RLS.
     pub fn set_file_loader(
         &mut self,
         file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
@@ -215,6 +218,7 @@ fn run_compiler(
             diagnostic_output,
             stderr: None,
             lint_caps: Default::default(),
+            parse_sess_created: None,
             register_lints: None,
             override_queries: None,
             make_codegen_backend: make_codegen_backend.take().unwrap(),
@@ -298,6 +302,7 @@ fn run_compiler(
         diagnostic_output,
         stderr: None,
         lint_caps: Default::default(),
+        parse_sess_created: None,
         register_lints: None,
         override_queries: None,
         make_codegen_backend: make_codegen_backend.unwrap(),
@@ -790,7 +795,7 @@ pub fn version(binary: &str, matches: &getopts::Matches) {
         println!("host: {}", config::host_triple());
         println!("release: {}", unw(util::release_str()));
         if cfg!(feature = "llvm") {
-            get_builtin_codegen_backend("llvm")().print_version();
+            get_builtin_codegen_backend(&None, "llvm")().print_version();
         }
     }
 }
@@ -840,7 +845,8 @@ the command line flag directly.
     );
 }
 
-fn describe_lints(sess: &Session, lint_store: &LintStore, loaded_plugins: bool) {
+/// Write to stdout lint command options, together with a list of all available lints
+pub fn describe_lints(sess: &Session, lint_store: &LintStore, loaded_plugins: bool) {
     println!(
         "
 Available lint options:
@@ -893,7 +899,12 @@ Available lint options:
     let print_lints = |lints: Vec<&Lint>| {
         for lint in lints {
             let name = lint.name_lower().replace("_", "-");
-            println!("    {}  {:7.7}  {}", padded(&name), lint.default_level.as_str(), lint.desc);
+            println!(
+                "    {}  {:7.7}  {}",
+                padded(&name),
+                lint.default_level(sess.edition()).as_str(),
+                lint.desc
+            );
         }
         println!("\n");
     };
@@ -1079,7 +1090,7 @@ pub fn handle_options(args: &[String]) -> Option<getopts::Matches> {
 
     if cg_flags.iter().any(|x| *x == "passes=list") {
         if cfg!(feature = "llvm") {
-            get_builtin_codegen_backend("llvm")().print_passes();
+            get_builtin_codegen_backend(&None, "llvm")().print_passes();
         }
         return None;
     }
diff --git a/compiler/rustc_driver/src/pretty.rs b/compiler/rustc_driver/src/pretty.rs
index 38c493a920d..e0c140b143b 100644
--- a/compiler/rustc_driver/src/pretty.rs
+++ b/compiler/rustc_driver/src/pretty.rs
@@ -9,12 +9,14 @@ use rustc_hir_pretty as pprust_hir;
 use rustc_middle::hir::map as hir_map;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_mir::util::{write_mir_graphviz, write_mir_pretty};
+use rustc_mir_build::thir;
 use rustc_session::config::{Input, PpAstTreeMode, PpHirMode, PpMode, PpSourceMode};
 use rustc_session::Session;
 use rustc_span::symbol::Ident;
 use rustc_span::FileName;
 
 use std::cell::Cell;
+use std::fmt::Write;
 use std::path::Path;
 
 pub use self::PpMode::*;
@@ -106,13 +108,6 @@ trait HirPrinterSupport<'hir>: pprust_hir::PpAnn {
     /// (Rust does not yet support upcasting from a trait object to
     /// an object for one of its super-traits.)
     fn pp_ann(&self) -> &dyn pprust_hir::PpAnn;
-
-    /// Computes an user-readable representation of a path, if possible.
-    fn node_path(&self, id: hir::HirId) -> Option<String> {
-        self.hir_map().and_then(|map| map.def_path_from_hir_id(id)).map(|path| {
-            path.data.into_iter().map(|elem| elem.data.to_string()).collect::<Vec<_>>().join("::")
-        })
-    }
 }
 
 struct NoAnn<'hir> {
@@ -325,10 +320,6 @@ impl<'tcx> HirPrinterSupport<'tcx> for TypedAnnotation<'tcx> {
     fn pp_ann(&self) -> &dyn pprust_hir::PpAnn {
         self
     }
-
-    fn node_path(&self, id: hir::HirId) -> Option<String> {
-        Some(self.tcx.def_path_str(self.tcx.hir().local_def_id(id).to_def_id()))
-    }
 }
 
 impl<'tcx> pprust_hir::PpAnn for TypedAnnotation<'tcx> {
@@ -484,18 +475,40 @@ fn print_with_analysis(
     ppm: PpMode,
     ofile: Option<&Path>,
 ) -> Result<(), ErrorReported> {
-    let mut out = Vec::new();
-
     tcx.analysis(LOCAL_CRATE)?;
 
-    match ppm {
-        Mir => write_mir_pretty(tcx, None, &mut out).unwrap(),
-        MirCFG => write_mir_graphviz(tcx, None, &mut out).unwrap(),
+    let out = match ppm {
+        Mir => {
+            let mut out = Vec::new();
+            write_mir_pretty(tcx, None, &mut out).unwrap();
+            String::from_utf8(out).unwrap()
+        }
+
+        MirCFG => {
+            let mut out = Vec::new();
+            write_mir_graphviz(tcx, None, &mut out).unwrap();
+            String::from_utf8(out).unwrap()
+        }
+
+        ThirTree => {
+            let mut out = String::new();
+            abort_on_err(rustc_typeck::check_crate(tcx), tcx.sess);
+            debug!("pretty printing THIR tree");
+            for did in tcx.body_owners() {
+                let hir = tcx.hir();
+                let body = hir.body(hir.body_owned_by(hir.local_def_id_to_hir_id(did)));
+                let arena = thir::Arena::default();
+                let thir =
+                    thir::build_thir(tcx, ty::WithOptConstParam::unknown(did), &arena, &body.value);
+                let _ = writeln!(out, "{:?}:\n{:#?}\n", did, thir);
+            }
+            out
+        }
+
         _ => unreachable!(),
-    }
+    };
 
-    let out = std::str::from_utf8(&out).unwrap();
-    write_or_print(out, ofile);
+    write_or_print(&out, ofile);
 
     Ok(())
 }
diff --git a/compiler/rustc_error_codes/src/error_codes/E0128.md b/compiler/rustc_error_codes/src/error_codes/E0128.md
index 6f8dfe3a73b..2ea8ae68ef8 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0128.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0128.md
@@ -7,7 +7,7 @@ struct Foo<T = U, U = ()> {
     field1: T,
     field2: U,
 }
-// error: type parameters with a default cannot use forward declared
+// error: generic parameters with a default cannot use forward declared
 //        identifiers
 ```
 
diff --git a/compiler/rustc_error_codes/src/error_codes/E0137.md b/compiler/rustc_error_codes/src/error_codes/E0137.md
index 0a02913d236..d4e19170f3f 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0137.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0137.md
@@ -1,8 +1,10 @@
+#### Note: this error code is no longer emitted by the compiler.
+
 More than one function was declared with the `#[main]` attribute.
 
 Erroneous code example:
 
-```compile_fail,E0137
+```compile_fail
 #![feature(main)]
 
 #[main]
@@ -16,7 +18,7 @@ This error indicates that the compiler found multiple functions with the
 `#[main]` attribute. This is an error because there must be a unique entry
 point into a Rust program. Example:
 
-```
+```compile_fail
 #![feature(main)]
 
 #[main]
diff --git a/compiler/rustc_error_codes/src/error_codes/E0404.md b/compiler/rustc_error_codes/src/error_codes/E0404.md
index 1360cc99afc..d6fa51e618c 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0404.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0404.md
@@ -8,14 +8,15 @@ struct Foo;
 struct Bar;
 
 impl Foo for Bar {} // error: `Foo` is not a trait
+fn baz<T: Foo>(t: T) {} // error: `Foo` is not a trait
 ```
 
 Another erroneous code example:
 
 ```compile_fail,E0404
-struct Foo;
+type Foo = Iterator<Item=String>;
 
-fn bar<T: Foo>(t: T) {} // error: `Foo` is not a trait
+fn bar<T: Foo>(t: T) {} // error: `Foo` is a type alias
 ```
 
 Please verify that the trait's name was not misspelled or that the right
@@ -30,14 +31,27 @@ struct Bar;
 impl Foo for Bar { // ok!
     // functions implementation
 }
+
+fn baz<T: Foo>(t: T) {} // ok!
 ```
 
-or:
+Alternatively, you could introduce a new trait with your desired restrictions
+as a super trait:
 
 ```
-trait Foo {
-    // some functions
-}
+# trait Foo {}
+# struct Bar;
+# impl Foo for Bar {}
+trait Qux: Foo {} // Anything that implements Qux also needs to implement Foo
+fn baz<T: Qux>(t: T) {} // also ok!
+```
+
+Finally, if you are on nightly and want to use a trait alias
+instead of a type alias, you should use `#![feature(trait_alias)]`:
+
+```
+#![feature(trait_alias)]
+trait Foo = Iterator<Item=String>;
 
 fn bar<T: Foo>(t: T) {} // ok!
 ```
diff --git a/compiler/rustc_error_codes/src/error_codes/E0554.md b/compiler/rustc_error_codes/src/error_codes/E0554.md
index e55fa4c6ede..3178bf21919 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0554.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0554.md
@@ -4,8 +4,8 @@ beta compilers will not comply.
 Erroneous code example:
 
 ```ignore (depends on release channel)
-#![feature(non_ascii_idents)] // error: `#![feature]` may not be used on the
-                              //        stable release channel
+#![feature(lang_items)] // error: `#![feature]` may not be used on the
+                        //        stable release channel
 ```
 
 If you need the feature, make sure to use a nightly release of the compiler
diff --git a/compiler/rustc_error_codes/src/error_codes/E0754.md b/compiler/rustc_error_codes/src/error_codes/E0754.md
index 9f4b19cfda6..acddb69aaba 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0754.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0754.md
@@ -3,7 +3,6 @@ A non-ASCII identifier was used in an invalid context.
 Erroneous code examples:
 
 ```compile_fail,E0754
-# #![feature(non_ascii_idents)]
 
 mod řųśť; // error!
 
@@ -17,8 +16,6 @@ Non-ASCII can be used as module names if it is inlined or if a `#[path]`
 attribute is specified. For example:
 
 ```
-# #![feature(non_ascii_idents)]
-
 mod řųśť { // ok!
     const IS_GREAT: bool = true;
 }
diff --git a/compiler/rustc_error_codes/src/lib.rs b/compiler/rustc_error_codes/src/lib.rs
index 14ddb3e2079..f2432f61653 100644
--- a/compiler/rustc_error_codes/src/lib.rs
+++ b/compiler/rustc_error_codes/src/lib.rs
@@ -1,5 +1,4 @@
-#![cfg_attr(bootstrap, deny(invalid_codeblock_attributes))]
-#![cfg_attr(not(bootstrap), deny(rustdoc::invalid_codeblock_attributes))]
+#![deny(rustdoc::invalid_codeblock_attributes)]
 //! This library is used to gather all error codes into one place,
 //! the goal being to make their maintenance easier.
 
diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs
index ce5b130dd97..b2f6a0c1014 100644
--- a/compiler/rustc_errors/src/diagnostic.rs
+++ b/compiler/rustc_errors/src/diagnostic.rs
@@ -69,10 +69,6 @@ impl DiagnosticStyledString {
     pub fn highlighted<S: Into<String>>(t: S) -> DiagnosticStyledString {
         DiagnosticStyledString(vec![StringPart::Highlighted(t.into())])
     }
-
-    pub fn content(&self) -> String {
-        self.0.iter().map(|x| x.content()).collect::<String>()
-    }
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -81,14 +77,6 @@ pub enum StringPart {
     Highlighted(String),
 }
 
-impl StringPart {
-    pub fn content(&self) -> &str {
-        match self {
-            &StringPart::Normal(ref s) | &StringPart::Highlighted(ref s) => s,
-        }
-    }
-}
-
 impl Diagnostic {
     pub fn new(level: Level, message: &str) -> Self {
         Diagnostic::new_with_code(level, None, message)
@@ -156,7 +144,7 @@ impl Diagnostic {
         self
     }
 
-    pub fn note_expected_found(
+    crate fn note_expected_found(
         &mut self,
         expected_label: &dyn fmt::Display,
         expected: DiagnosticStyledString,
@@ -166,7 +154,7 @@ impl Diagnostic {
         self.note_expected_found_extra(expected_label, expected, found_label, found, &"", &"")
     }
 
-    pub fn note_unsuccessful_coercion(
+    crate fn note_unsuccessful_coercion(
         &mut self,
         expected: DiagnosticStyledString,
         found: DiagnosticStyledString,
@@ -256,33 +244,33 @@ impl Diagnostic {
 
     /// Prints the span with a note above it.
     /// This is like [`Diagnostic::note()`], but it gets its own span.
-    pub fn span_note<S: Into<MultiSpan>>(&mut self, sp: S, msg: &str) -> &mut Self {
+    crate fn span_note<S: Into<MultiSpan>>(&mut self, sp: S, msg: &str) -> &mut Self {
         self.sub(Level::Note, msg, sp.into(), None);
         self
     }
 
     /// Add a warning attached to this diagnostic.
-    pub fn warn(&mut self, msg: &str) -> &mut Self {
+    crate fn warn(&mut self, msg: &str) -> &mut Self {
         self.sub(Level::Warning, msg, MultiSpan::new(), None);
         self
     }
 
     /// Prints the span with a warning above it.
     /// This is like [`Diagnostic::warn()`], but it gets its own span.
-    pub fn span_warn<S: Into<MultiSpan>>(&mut self, sp: S, msg: &str) -> &mut Self {
+    crate fn span_warn<S: Into<MultiSpan>>(&mut self, sp: S, msg: &str) -> &mut Self {
         self.sub(Level::Warning, msg, sp.into(), None);
         self
     }
 
     /// Add a help message attached to this diagnostic.
-    pub fn help(&mut self, msg: &str) -> &mut Self {
+    crate fn help(&mut self, msg: &str) -> &mut Self {
         self.sub(Level::Help, msg, MultiSpan::new(), None);
         self
     }
 
     /// Prints the span with some help above it.
     /// This is like [`Diagnostic::help()`], but it gets its own span.
-    pub fn span_help<S: Into<MultiSpan>>(&mut self, sp: S, msg: &str) -> &mut Self {
+    crate fn span_help<S: Into<MultiSpan>>(&mut self, sp: S, msg: &str) -> &mut Self {
         self.sub(Level::Help, msg, sp.into(), None);
         self
     }
@@ -311,36 +299,6 @@ impl Diagnostic {
         self
     }
 
-    /// Show multiple suggestions that have multiple parts.
-    /// See also [`Diagnostic::multipart_suggestion()`].
-    pub fn multipart_suggestions(
-        &mut self,
-        msg: &str,
-        suggestions: Vec<Vec<(Span, String)>>,
-        applicability: Applicability,
-    ) -> &mut Self {
-        assert!(!suggestions.is_empty());
-        for s in &suggestions {
-            assert!(!s.is_empty());
-        }
-        self.suggestions.push(CodeSuggestion {
-            substitutions: suggestions
-                .into_iter()
-                .map(|suggestion| Substitution {
-                    parts: suggestion
-                        .into_iter()
-                        .map(|(span, snippet)| SubstitutionPart { snippet, span })
-                        .collect(),
-                })
-                .collect(),
-            msg: msg.to_owned(),
-            style: SuggestionStyle::ShowCode,
-            applicability,
-            tool_metadata: Default::default(),
-        });
-        self
-    }
-
     /// Prints out a message with for a multipart suggestion without showing the suggested code.
     ///
     /// This is intended to be used for suggestions that are obvious in what the changes need to
@@ -567,7 +525,7 @@ impl Diagnostic {
         self.code.clone()
     }
 
-    pub fn set_primary_message<M: Into<String>>(&mut self, msg: M) -> &mut Self {
+    crate fn set_primary_message<M: Into<String>>(&mut self, msg: M) -> &mut Self {
         self.message[0] = (msg.into(), Style::NoStyle);
         self
     }
@@ -582,6 +540,8 @@ impl Diagnostic {
 
     /// Convenience function for internal use, clients should use one of the
     /// public methods above.
+    ///
+    /// Used by `proc_macro_server` for implementing `server::Diagnostic`.
     pub fn sub(
         &mut self,
         level: Level,
diff --git a/compiler/rustc_errors/src/diagnostic_builder.rs b/compiler/rustc_errors/src/diagnostic_builder.rs
index c09cce21bf2..282877d5dd1 100644
--- a/compiler/rustc_errors/src/diagnostic_builder.rs
+++ b/compiler/rustc_errors/src/diagnostic_builder.rs
@@ -30,15 +30,6 @@ struct DiagnosticBuilderInner<'a> {
     allow_suggestions: bool,
 }
 
-/// This is a helper macro for [`forward!`] that allows automatically adding documentation
-/// that uses tokens from [`forward!`]'s input.
-macro_rules! forward_inner_docs {
-    ($e:expr => $i:item) => {
-        #[doc = $e]
-        $i
-    };
-}
-
 /// In general, the `DiagnosticBuilder` uses deref to allow access to
 /// the fields and methods of the embedded `diagnostic` in a
 /// transparent way. *However,* many of the methods are intended to
@@ -54,11 +45,11 @@ macro_rules! forward {
         pub fn $n:ident(&self, $($name:ident: $ty:ty),* $(,)?) -> &Self
     ) => {
         $(#[$attrs])*
-        forward_inner_docs!(concat!("See [`Diagnostic::", stringify!($n), "()`].") =>
+        #[doc = concat!("See [`Diagnostic::", stringify!($n), "()`].")]
         pub fn $n(&self, $($name: $ty),*) -> &Self {
             self.diagnostic.$n($($name),*);
             self
-        });
+        }
     };
 
     // Forward pattern for &mut self -> &mut Self
@@ -67,11 +58,11 @@ macro_rules! forward {
         pub fn $n:ident(&mut self, $($name:ident: $ty:ty),* $(,)?) -> &mut Self
     ) => {
         $(#[$attrs])*
-        forward_inner_docs!(concat!("See [`Diagnostic::", stringify!($n), "()`].") =>
+        #[doc = concat!("See [`Diagnostic::", stringify!($n), "()`].")]
         pub fn $n(&mut self, $($name: $ty),*) -> &mut Self {
             self.0.diagnostic.$n($($name),*);
             self
-        });
+        }
     };
 
     // Forward pattern for &mut self -> &mut Self, with generic parameters.
@@ -84,11 +75,11 @@ macro_rules! forward {
         ) -> &mut Self
     ) => {
         $(#[$attrs])*
-        forward_inner_docs!(concat!("See [`Diagnostic::", stringify!($n), "()`].") =>
+        #[doc = concat!("See [`Diagnostic::", stringify!($n), "()`].")]
         pub fn $n<$($generic: $bound),*>(&mut self, $($name: $ty),*) -> &mut Self {
             self.0.diagnostic.$n($($name),*);
             self
-        });
+        }
     };
 }
 
@@ -166,19 +157,6 @@ impl<'a> DiagnosticBuilder<'a> {
         buffered_diagnostics.extend(self.into_diagnostic().map(|(diag, _)| diag));
     }
 
-    /// Convenience function for internal use, clients should use one of the
-    /// span_* methods instead.
-    pub fn sub<S: Into<MultiSpan>>(
-        &mut self,
-        level: Level,
-        message: &str,
-        span: Option<S>,
-    ) -> &mut Self {
-        let span = span.map(|s| s.into()).unwrap_or_else(MultiSpan::new);
-        self.0.diagnostic.sub(level, message, span, None);
-        self
-    }
-
     /// Delay emission of this diagnostic as a bug.
     ///
     /// This can be useful in contexts where an error indicates a bug but
@@ -279,20 +257,6 @@ impl<'a> DiagnosticBuilder<'a> {
         self
     }
 
-    /// See [`Diagnostic::multipart_suggestions()`].
-    pub fn multipart_suggestions(
-        &mut self,
-        msg: &str,
-        suggestions: Vec<Vec<(Span, String)>>,
-        applicability: Applicability,
-    ) -> &mut Self {
-        if !self.0.allow_suggestions {
-            return self;
-        }
-        self.0.diagnostic.multipart_suggestions(msg, suggestions, applicability);
-        self
-    }
-
     /// See [`Diagnostic::tool_only_multipart_suggestion()`].
     pub fn tool_only_multipart_suggestion(
         &mut self,
diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs
index 9b6f67166bd..a58caf2667b 100644
--- a/compiler/rustc_errors/src/emitter.rs
+++ b/compiler/rustc_errors/src/emitter.rs
@@ -195,6 +195,9 @@ pub trait Emitter {
 
     fn emit_future_breakage_report(&mut self, _diags: Vec<(FutureBreakage, Diagnostic)>) {}
 
+    /// Emit list of unused externs
+    fn emit_unused_externs(&mut self, _lint_level: &str, _unused_externs: &[&str]) {}
+
     /// Checks if should show explanations about "rustc --explain"
     fn should_show_explain(&self) -> bool {
         true
@@ -434,9 +437,15 @@ pub trait Emitter {
         span: &mut MultiSpan,
         children: &mut Vec<SubDiagnostic>,
     ) {
+        let source_map = if let Some(ref sm) = source_map {
+            sm
+        } else {
+            return;
+        };
         debug!("fix_multispans_in_extern_macros: before: span={:?} children={:?}", span, children);
-        for span in iter::once(&mut *span).chain(children.iter_mut().map(|child| &mut child.span)) {
-            self.fix_multispan_in_extern_macros(source_map, span);
+        self.fix_multispan_in_extern_macros(source_map, span);
+        for child in children.iter_mut() {
+            self.fix_multispan_in_extern_macros(source_map, &mut child.span);
         }
         debug!("fix_multispans_in_extern_macros: after: span={:?} children={:?}", span, children);
     }
@@ -444,16 +453,7 @@ pub trait Emitter {
     // This "fixes" MultiSpans that contain `Span`s pointing to locations inside of external macros.
     // Since these locations are often difficult to read,
     // we move these spans from the external macros to their corresponding use site.
-    fn fix_multispan_in_extern_macros(
-        &self,
-        source_map: &Option<Lrc<SourceMap>>,
-        span: &mut MultiSpan,
-    ) {
-        let sm = match source_map {
-            Some(ref sm) => sm,
-            None => return,
-        };
-
+    fn fix_multispan_in_extern_macros(&self, source_map: &Lrc<SourceMap>, span: &mut MultiSpan) {
         // First, find all the spans in external macros and point instead at their use site.
         let replacements: Vec<(Span, Span)> = span
             .primary_spans()
@@ -461,7 +461,7 @@ pub trait Emitter {
             .copied()
             .chain(span.span_labels().iter().map(|sp_label| sp_label.span))
             .filter_map(|sp| {
-                if !sp.is_dummy() && sm.is_imported(sp) {
+                if !sp.is_dummy() && source_map.is_imported(sp) {
                     let maybe_callsite = sp.source_callsite();
                     if sp != maybe_callsite {
                         return Some((sp, maybe_callsite));
@@ -1232,7 +1232,6 @@ impl EmitterWriter {
         is_secondary: bool,
     ) -> io::Result<()> {
         let mut buffer = StyledBuffer::new();
-        let header_style = if is_secondary { Style::HeaderMsg } else { Style::MainHeaderMsg };
 
         if !msp.has_primary_spans() && !msp.has_span_labels() && is_secondary && !self.short_message
         {
@@ -1257,11 +1256,12 @@ impl EmitterWriter {
                 buffer.append(0, &code, Style::Level(*level));
                 buffer.append(0, "]", Style::Level(*level));
             }
+            let header_style = if is_secondary { Style::HeaderMsg } else { Style::MainHeaderMsg };
             if *level != Level::FailureNote {
                 buffer.append(0, ": ", header_style);
             }
             for &(ref text, _) in msg.iter() {
-                buffer.append(0, text, header_style);
+                buffer.append(0, &replace_tabs(text), header_style);
             }
         }
 
@@ -1470,9 +1470,7 @@ impl EmitterWriter {
                     let mut to_add = FxHashMap::default();
 
                     for (depth, style) in depths {
-                        if multilines.get(&depth).is_some() {
-                            multilines.remove(&depth);
-                        } else {
+                        if multilines.remove(&depth).is_none() {
                             to_add.insert(depth, style);
                         }
                     }
@@ -1726,14 +1724,13 @@ impl EmitterWriter {
                     if !self.short_message {
                         draw_col_separator_no_space(&mut buffer, 0, max_line_num_len + 1);
                     }
-                    match emit_to_destination(
+                    if let Err(e) = emit_to_destination(
                         &buffer.render(),
                         level,
                         &mut self.dst,
                         self.short_message,
                     ) {
-                        Ok(()) => (),
-                        Err(e) => panic!("failed to emit error: {}", e),
+                        panic!("failed to emit error: {}", e)
                     }
                 }
                 if !self.short_message {
@@ -2220,9 +2217,7 @@ pub fn is_case_difference(sm: &SourceMap, suggested: &str, sp: Span) -> bool {
     };
     let ascii_confusables = &['c', 'f', 'i', 'k', 'o', 's', 'u', 'v', 'w', 'x', 'y', 'z'];
     // All the chars that differ in capitalization are confusable (above):
-    let confusable = found
-        .chars()
-        .zip(suggested.chars())
+    let confusable = iter::zip(found.chars(), suggested.chars())
         .filter(|(f, s)| f != s)
         .all(|(f, s)| (ascii_confusables.contains(&f) || ascii_confusables.contains(&s)));
     confusable && found.to_lowercase() == suggested.to_lowercase()
diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs
index c27b39a9d62..40277006462 100644
--- a/compiler/rustc_errors/src/json.rs
+++ b/compiler/rustc_errors/src/json.rs
@@ -159,6 +159,19 @@ impl Emitter for JsonEmitter {
         }
     }
 
+    fn emit_unused_externs(&mut self, lint_level: &str, unused_externs: &[&str]) {
+        let data = UnusedExterns { lint_level, unused_extern_names: unused_externs };
+        let result = if self.pretty {
+            writeln!(&mut self.dst, "{}", as_pretty_json(&data))
+        } else {
+            writeln!(&mut self.dst, "{}", as_json(&data))
+        }
+        .and_then(|_| self.dst.flush());
+        if let Err(e) = result {
+            panic!("failed to print unused externs: {:?}", e);
+        }
+    }
+
     fn source_map(&self) -> Option<&Lrc<SourceMap>> {
         Some(&self.sm)
     }
@@ -322,6 +335,18 @@ struct FutureIncompatReport {
     future_incompat_report: Vec<FutureBreakageItem>,
 }
 
+// NOTE: Keep this in sync with the equivalent structs in rustdoc's
+// doctest component (as well as cargo).
+// We could unify this struct the one in rustdoc but they have different
+// ownership semantics, so doing so would create wasteful allocations.
+#[derive(Encodable)]
+struct UnusedExterns<'a, 'b, 'c> {
+    /// The severity level of the unused dependencies lint
+    lint_level: &'a str,
+    /// List of unused externs by their names.
+    unused_extern_names: &'b [&'c str],
+}
+
 impl Diagnostic {
     fn from_errors_diagnostic(diag: &crate::Diagnostic, je: &JsonEmitter) -> Diagnostic {
         let sugg = diag.suggestions.iter().map(|sugg| Diagnostic {
@@ -493,7 +518,7 @@ impl DiagnosticSpanLine {
         h_end: usize,
     ) -> DiagnosticSpanLine {
         DiagnosticSpanLine {
-            text: sf.get_line(index).map_or(String::new(), |l| l.into_owned()),
+            text: sf.get_line(index).map_or_else(String::new, |l| l.into_owned()),
             highlight_start: h_start,
             highlight_end: h_end,
         }
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index a0be7442d59..f1a31f0d4f5 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -5,6 +5,8 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(crate_visibility_modifier)]
 #![feature(backtrace)]
+#![feature(extended_key_value_attributes)]
+#![feature(iter_zip)]
 #![feature(nll)]
 
 #[macro_use]
@@ -52,7 +54,7 @@ pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;
 
 // `PResult` is used a lot. Make sure it doesn't unintentionally get bigger.
 // (See also the comment on `DiagnosticBuilderInner`.)
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(PResult<'_, bool>, 16);
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Encodable, Decodable)]
@@ -319,7 +321,7 @@ struct HandlerInner {
 
     /// This set contains the `DiagnosticId` of all emitted diagnostics to avoid
     /// emitting the same diagnostic with extended help (`--teach`) twice, which
-    /// would be uneccessary repetition.
+    /// would be unnecessary repetition.
     taught_diagnostics: FxHashSet<DiagnosticId>,
 
     /// Used to suggest rustc --explain <error code>
@@ -689,10 +691,6 @@ impl Handler {
         db
     }
 
-    pub fn failure(&self, msg: &str) {
-        self.inner.borrow_mut().failure(msg);
-    }
-
     pub fn fatal(&self, msg: &str) -> FatalError {
         self.inner.borrow_mut().fatal(msg)
     }
@@ -767,6 +765,10 @@ impl Handler {
         self.inner.borrow_mut().emitter.emit_future_breakage_report(diags)
     }
 
+    pub fn emit_unused_externs(&self, lint_level: &str, unused_externs: &[&str]) {
+        self.inner.borrow_mut().emit_unused_externs(lint_level, unused_externs)
+    }
+
     pub fn delay_as_bug(&self, diagnostic: Diagnostic) {
         self.inner.borrow_mut().delay_as_bug(diagnostic)
     }
@@ -841,6 +843,10 @@ impl HandlerInner {
         self.emitter.emit_artifact_notification(path, artifact_type);
     }
 
+    fn emit_unused_externs(&mut self, lint_level: &str, unused_externs: &[&str]) {
+        self.emitter.emit_unused_externs(lint_level, unused_externs);
+    }
+
     fn treat_err_as_bug(&self) -> bool {
         self.flags.treat_err_as_bug.map_or(false, |c| self.err_count() >= c.get())
     }
diff --git a/compiler/rustc_errors/src/registry.rs b/compiler/rustc_errors/src/registry.rs
index b1d770d5bd5..da764d993bb 100644
--- a/compiler/rustc_errors/src/registry.rs
+++ b/compiler/rustc_errors/src/registry.rs
@@ -13,10 +13,6 @@ impl Registry {
         Registry { long_descriptions: long_descriptions.iter().copied().collect() }
     }
 
-    /// This will panic if an invalid error code is passed in
-    pub fn find_description(&self, code: &str) -> Option<&'static str> {
-        self.long_descriptions[code]
-    }
     /// Returns `InvalidErrorCode` if the code requested does not exist in the
     /// registry. Otherwise, returns an `Option` where `None` means the error
     /// code is valid but has no extended information.
diff --git a/compiler/rustc_errors/src/snippet.rs b/compiler/rustc_errors/src/snippet.rs
index acb88e57db5..3fe02bd0cee 100644
--- a/compiler/rustc_errors/src/snippet.rs
+++ b/compiler/rustc_errors/src/snippet.rs
@@ -121,16 +121,6 @@ impl Annotation {
         matches!(self.annotation_type, AnnotationType::MultilineLine(_))
     }
 
-    pub fn is_multiline(&self) -> bool {
-        matches!(
-            self.annotation_type,
-            AnnotationType::Multiline(_)
-                | AnnotationType::MultilineStart(_)
-                | AnnotationType::MultilineLine(_)
-                | AnnotationType::MultilineEnd(_)
-        )
-    }
-
     pub fn len(&self) -> usize {
         // Account for usize underflows
         if self.end_col > self.start_col {
diff --git a/compiler/rustc_errors/src/styled_buffer.rs b/compiler/rustc_errors/src/styled_buffer.rs
index ef71ee36ea3..ec122e7be6e 100644
--- a/compiler/rustc_errors/src/styled_buffer.rs
+++ b/compiler/rustc_errors/src/styled_buffer.rs
@@ -1,6 +1,7 @@
 // Code for creating styled buffers
 
 use crate::snippet::{Style, StyledString};
+use std::iter;
 
 #[derive(Debug)]
 pub struct StyledBuffer {
@@ -20,11 +21,11 @@ impl StyledBuffer {
         let mut output: Vec<Vec<StyledString>> = vec![];
         let mut styled_vec: Vec<StyledString> = vec![];
 
-        for (row, row_style) in self.text.iter().zip(&self.styles) {
+        for (row, row_style) in iter::zip(&self.text, &self.styles) {
             let mut current_style = Style::NoStyle;
             let mut current_text = String::new();
 
-            for (&c, &s) in row.iter().zip(row_style) {
+            for (&c, &s) in iter::zip(row, row_style) {
                 if s != current_style {
                     if !current_text.is_empty() {
                         styled_vec.push(StyledString { text: current_text, style: current_style });
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index 929cc56294d..59505842816 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -1,15 +1,17 @@
 use crate::expand::{self, AstFragment, Invocation};
-use crate::module::DirectoryOwnership;
+use crate::module::DirOwnership;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Nonterminal};
-use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, LazyTokenStream, TokenStream};
+use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream};
 use rustc_ast::visit::{AssocCtxt, Visitor};
-use rustc_ast::{self as ast, AstLike, Attribute, NodeId, PatKind};
+use rustc_ast::{self as ast, AstLike, Attribute, Item, NodeId, PatKind};
 use rustc_attr::{self as attr, Deprecation, Stability};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagnosticBuilder, ErrorReported};
+use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
+use rustc_lint_defs::BuiltinLintDiagnostics;
 use rustc_parse::{self, nt_to_tokenstream, parser, MACRO_ARGUMENTS};
 use rustc_session::{parse::ParseSess, Limit, Session};
 use rustc_span::def_id::DefId;
@@ -36,34 +38,34 @@ pub enum Annotatable {
     Stmt(P<ast::Stmt>),
     Expr(P<ast::Expr>),
     Arm(ast::Arm),
-    Field(ast::Field),
-    FieldPat(ast::FieldPat),
+    ExprField(ast::ExprField),
+    PatField(ast::PatField),
     GenericParam(ast::GenericParam),
     Param(ast::Param),
-    StructField(ast::StructField),
+    FieldDef(ast::FieldDef),
     Variant(ast::Variant),
 }
 
-impl AstLike for Annotatable {
-    fn attrs(&self) -> &[Attribute] {
+impl Annotatable {
+    pub fn span(&self) -> Span {
         match *self {
-            Annotatable::Item(ref item) => &item.attrs,
-            Annotatable::TraitItem(ref trait_item) => &trait_item.attrs,
-            Annotatable::ImplItem(ref impl_item) => &impl_item.attrs,
-            Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs,
-            Annotatable::Stmt(ref stmt) => stmt.attrs(),
-            Annotatable::Expr(ref expr) => &expr.attrs,
-            Annotatable::Arm(ref arm) => &arm.attrs,
-            Annotatable::Field(ref field) => &field.attrs,
-            Annotatable::FieldPat(ref fp) => &fp.attrs,
-            Annotatable::GenericParam(ref gp) => &gp.attrs,
-            Annotatable::Param(ref p) => &p.attrs,
-            Annotatable::StructField(ref sf) => &sf.attrs,
-            Annotatable::Variant(ref v) => &v.attrs(),
+            Annotatable::Item(ref item) => item.span,
+            Annotatable::TraitItem(ref trait_item) => trait_item.span,
+            Annotatable::ImplItem(ref impl_item) => impl_item.span,
+            Annotatable::ForeignItem(ref foreign_item) => foreign_item.span,
+            Annotatable::Stmt(ref stmt) => stmt.span,
+            Annotatable::Expr(ref expr) => expr.span,
+            Annotatable::Arm(ref arm) => arm.span,
+            Annotatable::ExprField(ref field) => field.span,
+            Annotatable::PatField(ref fp) => fp.pat.span,
+            Annotatable::GenericParam(ref gp) => gp.ident.span,
+            Annotatable::Param(ref p) => p.span,
+            Annotatable::FieldDef(ref sf) => sf.span,
+            Annotatable::Variant(ref v) => v.span,
         }
     }
 
-    fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
+    pub fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         match self {
             Annotatable::Item(item) => item.visit_attrs(f),
             Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f),
@@ -72,39 +74,15 @@ impl AstLike for Annotatable {
             Annotatable::Stmt(stmt) => stmt.visit_attrs(f),
             Annotatable::Expr(expr) => expr.visit_attrs(f),
             Annotatable::Arm(arm) => arm.visit_attrs(f),
-            Annotatable::Field(field) => field.visit_attrs(f),
-            Annotatable::FieldPat(fp) => fp.visit_attrs(f),
+            Annotatable::ExprField(field) => field.visit_attrs(f),
+            Annotatable::PatField(fp) => fp.visit_attrs(f),
             Annotatable::GenericParam(gp) => gp.visit_attrs(f),
             Annotatable::Param(p) => p.visit_attrs(f),
-            Annotatable::StructField(sf) => sf.visit_attrs(f),
+            Annotatable::FieldDef(sf) => sf.visit_attrs(f),
             Annotatable::Variant(v) => v.visit_attrs(f),
         }
     }
 
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        panic!("Called finalize_tokens on an Annotatable: {:?}", tokens);
-    }
-}
-
-impl Annotatable {
-    pub fn span(&self) -> Span {
-        match *self {
-            Annotatable::Item(ref item) => item.span,
-            Annotatable::TraitItem(ref trait_item) => trait_item.span,
-            Annotatable::ImplItem(ref impl_item) => impl_item.span,
-            Annotatable::ForeignItem(ref foreign_item) => foreign_item.span,
-            Annotatable::Stmt(ref stmt) => stmt.span,
-            Annotatable::Expr(ref expr) => expr.span,
-            Annotatable::Arm(ref arm) => arm.span,
-            Annotatable::Field(ref field) => field.span,
-            Annotatable::FieldPat(ref fp) => fp.pat.span,
-            Annotatable::GenericParam(ref gp) => gp.ident.span,
-            Annotatable::Param(ref p) => p.span,
-            Annotatable::StructField(ref sf) => sf.span,
-            Annotatable::Variant(ref v) => v.span,
-        }
-    }
-
     pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
         match self {
             Annotatable::Item(item) => visitor.visit_item(item),
@@ -114,16 +92,16 @@ impl Annotatable {
             Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt),
             Annotatable::Expr(expr) => visitor.visit_expr(expr),
             Annotatable::Arm(arm) => visitor.visit_arm(arm),
-            Annotatable::Field(field) => visitor.visit_field(field),
-            Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp),
+            Annotatable::ExprField(field) => visitor.visit_expr_field(field),
+            Annotatable::PatField(fp) => visitor.visit_pat_field(fp),
             Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp),
             Annotatable::Param(p) => visitor.visit_param(p),
-            Annotatable::StructField(sf) => visitor.visit_struct_field(sf),
+            Annotatable::FieldDef(sf) => visitor.visit_field_def(sf),
             Annotatable::Variant(v) => visitor.visit_variant(v),
         }
     }
 
-    crate fn into_nonterminal(self) -> Nonterminal {
+    pub fn into_nonterminal(self) -> Nonterminal {
         match self {
             Annotatable::Item(item) => token::NtItem(item),
             Annotatable::TraitItem(item) | Annotatable::ImplItem(item) => {
@@ -135,20 +113,17 @@ impl Annotatable {
             Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
             Annotatable::Expr(expr) => token::NtExpr(expr),
             Annotatable::Arm(..)
-            | Annotatable::Field(..)
-            | Annotatable::FieldPat(..)
+            | Annotatable::ExprField(..)
+            | Annotatable::PatField(..)
             | Annotatable::GenericParam(..)
             | Annotatable::Param(..)
-            | Annotatable::StructField(..)
+            | Annotatable::FieldDef(..)
             | Annotatable::Variant(..) => panic!("unexpected annotatable"),
         }
     }
 
     crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
-        // Tokens of an attribute target may be invalidated by some outer `#[derive]` performing
-        // "full configuration" (attributes following derives on the same item should be the most
-        // common case), that's why synthesizing tokens is allowed.
-        nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::Yes)
+        nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::No)
     }
 
     pub fn expect_item(self) -> P<ast::Item> {
@@ -200,16 +175,16 @@ impl Annotatable {
         }
     }
 
-    pub fn expect_field(self) -> ast::Field {
+    pub fn expect_expr_field(self) -> ast::ExprField {
         match self {
-            Annotatable::Field(field) => field,
+            Annotatable::ExprField(field) => field,
             _ => panic!("expected field"),
         }
     }
 
-    pub fn expect_field_pattern(self) -> ast::FieldPat {
+    pub fn expect_pat_field(self) -> ast::PatField {
         match self {
-            Annotatable::FieldPat(fp) => fp,
+            Annotatable::PatField(fp) => fp,
             _ => panic!("expected field pattern"),
         }
     }
@@ -228,9 +203,9 @@ impl Annotatable {
         }
     }
 
-    pub fn expect_struct_field(self) -> ast::StructField {
+    pub fn expect_field_def(self) -> ast::FieldDef {
         match self {
-            Annotatable::StructField(sf) => sf,
+            Annotatable::FieldDef(sf) => sf,
             _ => panic!("expected struct field"),
         }
     }
@@ -416,11 +391,11 @@ pub trait MacResult {
         None
     }
 
-    fn make_fields(self: Box<Self>) -> Option<SmallVec<[ast::Field; 1]>> {
+    fn make_expr_fields(self: Box<Self>) -> Option<SmallVec<[ast::ExprField; 1]>> {
         None
     }
 
-    fn make_field_patterns(self: Box<Self>) -> Option<SmallVec<[ast::FieldPat; 1]>> {
+    fn make_pat_fields(self: Box<Self>) -> Option<SmallVec<[ast::PatField; 1]>> {
         None
     }
 
@@ -432,7 +407,7 @@ pub trait MacResult {
         None
     }
 
-    fn make_struct_fields(self: Box<Self>) -> Option<SmallVec<[ast::StructField; 1]>> {
+    fn make_field_defs(self: Box<Self>) -> Option<SmallVec<[ast::FieldDef; 1]>> {
         None
     }
 
@@ -616,11 +591,11 @@ impl MacResult for DummyResult {
         Some(SmallVec::new())
     }
 
-    fn make_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::Field; 1]>> {
+    fn make_expr_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::ExprField; 1]>> {
         Some(SmallVec::new())
     }
 
-    fn make_field_patterns(self: Box<DummyResult>) -> Option<SmallVec<[ast::FieldPat; 1]>> {
+    fn make_pat_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::PatField; 1]>> {
         Some(SmallVec::new())
     }
 
@@ -632,7 +607,7 @@ impl MacResult for DummyResult {
         Some(SmallVec::new())
     }
 
-    fn make_struct_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::StructField; 1]>> {
+    fn make_field_defs(self: Box<DummyResult>) -> Option<SmallVec<[ast::FieldDef; 1]>> {
         Some(SmallVec::new())
     }
 
@@ -761,7 +736,7 @@ impl SyntaxExtension {
         attrs: &[ast::Attribute],
     ) -> SyntaxExtension {
         let allow_internal_unstable =
-            Some(attr::allow_internal_unstable(sess, &attrs).collect::<Vec<Symbol>>().into());
+            attr::allow_internal_unstable(sess, &attrs).collect::<Vec<Symbol>>();
 
         let mut local_inner_macros = false;
         if let Some(macro_export) = sess.find_by_name(attrs, sym::macro_export) {
@@ -789,7 +764,8 @@ impl SyntaxExtension {
         SyntaxExtension {
             kind,
             span,
-            allow_internal_unstable,
+            allow_internal_unstable: (!allow_internal_unstable.is_empty())
+                .then(|| allow_internal_unstable.into()),
             allow_internal_unsafe: sess.contains_name(attrs, sym::allow_internal_unsafe),
             local_inner_macros,
             stability: stability.map(|(s, _)| s),
@@ -851,6 +827,8 @@ impl SyntaxExtension {
 /// Error type that denotes indeterminacy.
 pub struct Indeterminate;
 
+pub type DeriveResolutions = Vec<(ast::Path, Option<Lrc<SyntaxExtension>>)>;
+
 pub trait ResolverExpand {
     fn next_node_id(&mut self) -> NodeId;
 
@@ -887,23 +865,36 @@ pub trait ResolverExpand {
     fn resolve_derives(
         &mut self,
         expn_id: ExpnId,
-        derives: Vec<ast::Path>,
         force: bool,
+        derive_paths: &dyn Fn() -> DeriveResolutions,
     ) -> Result<(), Indeterminate>;
     /// Take resolutions for paths inside the `#[derive(...)]` attribute with the given `ExpnId`
     /// back from resolver.
-    fn take_derive_resolutions(
-        &mut self,
-        expn_id: ExpnId,
-    ) -> Option<Vec<(Lrc<SyntaxExtension>, ast::Path)>>;
+    fn take_derive_resolutions(&mut self, expn_id: ExpnId) -> Option<DeriveResolutions>;
     /// Path resolution logic for `#[cfg_accessible(path)]`.
     fn cfg_accessible(&mut self, expn_id: ExpnId, path: &ast::Path) -> Result<bool, Indeterminate>;
 }
 
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct ModuleData {
+    /// Path to the module starting from the crate name, like `my_crate::foo::bar`.
     pub mod_path: Vec<Ident>,
-    pub directory: PathBuf,
+    /// Stack of paths to files loaded by out-of-line module items,
+    /// used to detect and report recursive module inclusions.
+    pub file_path_stack: Vec<PathBuf>,
+    /// Directory to search child module files in,
+    /// often (but not necessarily) the parent of the top file path on the `file_path_stack`.
+    pub dir_path: PathBuf,
+}
+
+impl ModuleData {
+    pub fn with_dir_path(&self, dir_path: PathBuf) -> ModuleData {
+        ModuleData {
+            mod_path: self.mod_path.clone(),
+            file_path_stack: self.file_path_stack.clone(),
+            dir_path,
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -911,10 +902,13 @@ pub struct ExpansionData {
     pub id: ExpnId,
     pub depth: usize,
     pub module: Rc<ModuleData>,
-    pub directory_ownership: DirectoryOwnership,
+    pub dir_ownership: DirOwnership,
     pub prior_type_ascription: Option<(Span, bool)>,
 }
 
+type OnExternModLoaded<'a> =
+    Option<&'a dyn Fn(Ident, Vec<Attribute>, Vec<P<Item>>, Span) -> (Vec<Attribute>, Vec<P<Item>>)>;
+
 /// One of these is made during expansion and incrementally updated as we go;
 /// when a macro expansion occurs, the resulting nodes have the `backtrace()
 /// -> expn_data` of their expansion context stored into their span.
@@ -932,7 +926,7 @@ pub struct ExtCtxt<'a> {
     /// Called directly after having parsed an external `mod foo;` in expansion.
     ///
     /// `Ident` is the module name.
-    pub(super) extern_mod_loaded: Option<&'a dyn Fn(&ast::Crate, Ident)>,
+    pub(super) extern_mod_loaded: OnExternModLoaded<'a>,
 }
 
 impl<'a> ExtCtxt<'a> {
@@ -940,7 +934,7 @@ impl<'a> ExtCtxt<'a> {
         sess: &'a Session,
         ecfg: expand::ExpansionConfig<'a>,
         resolver: &'a mut dyn ResolverExpand,
-        extern_mod_loaded: Option<&'a dyn Fn(&ast::Crate, Ident)>,
+        extern_mod_loaded: OnExternModLoaded<'a>,
     ) -> ExtCtxt<'a> {
         ExtCtxt {
             sess,
@@ -952,8 +946,8 @@ impl<'a> ExtCtxt<'a> {
             current_expansion: ExpansionData {
                 id: ExpnId::root(),
                 depth: 0,
-                module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }),
-                directory_ownership: DirectoryOwnership::Owned { relative: None },
+                module: Default::default(),
+                dir_ownership: DirOwnership::Owned { relative: None },
                 prior_type_ascription: None,
             },
             force_mode: false,
@@ -1207,3 +1201,41 @@ pub fn get_exprs_from_tts(
     }
     Some(es)
 }
+
+/// This nonterminal looks like some specific enums from
+/// `proc-macro-hack` and `procedural-masquerade` crates.
+/// We need to maintain some special pretty-printing behavior for them due to incorrect
+/// asserts in old versions of those crates and their wide use in the ecosystem.
+/// See issue #73345 for more details.
+/// FIXME(#73933): Remove this eventually.
+pub(crate) fn pretty_printing_compatibility_hack(nt: &Nonterminal, sess: &ParseSess) -> bool {
+    let item = match nt {
+        Nonterminal::NtItem(item) => item,
+        Nonterminal::NtStmt(stmt) => match &stmt.kind {
+            ast::StmtKind::Item(item) => item,
+            _ => return false,
+        },
+        _ => return false,
+    };
+
+    let name = item.ident.name;
+    if name == sym::ProceduralMasqueradeDummyType {
+        if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
+            if let [variant] = &*enum_def.variants {
+                if variant.ident.name == sym::Input {
+                    sess.buffer_lint_with_diagnostic(
+                        &PROC_MACRO_BACK_COMPAT,
+                        item.ident.span,
+                        ast::CRATE_NODE_ID,
+                        "using `procedural-masquerade` crate",
+                        BuiltinLintDiagnostics::ProcMacroBackCompat(
+                        "The `procedural-masquerade` crate has been unnecessary since Rust 1.30.0. \
+                        Versions of this crate below 0.1.7 will eventually stop compiling.".to_string())
+                    );
+                    return true;
+                }
+            }
+        }
+    }
+    false
+}
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index fe67b401fcc..cb8b9398283 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -253,22 +253,11 @@ impl<'a> ExtCtxt<'a> {
         let pathexpr = self.expr_path(self.path_global(sp, fn_path));
         self.expr_call(sp, pathexpr, args)
     }
-    pub fn expr_method_call(
-        &self,
-        span: Span,
-        expr: P<ast::Expr>,
-        ident: Ident,
-        mut args: Vec<P<ast::Expr>>,
-    ) -> P<ast::Expr> {
-        args.insert(0, expr);
-        let segment = ast::PathSegment::from_ident(ident.with_span_pos(span));
-        self.expr(span, ast::ExprKind::MethodCall(segment, args, span))
-    }
     pub fn expr_block(&self, b: P<ast::Block>) -> P<ast::Expr> {
         self.expr(b.span, ast::ExprKind::Block(b, None))
     }
-    pub fn field_imm(&self, span: Span, ident: Ident, e: P<ast::Expr>) -> ast::Field {
-        ast::Field {
+    pub fn field_imm(&self, span: Span, ident: Ident, e: P<ast::Expr>) -> ast::ExprField {
+        ast::ExprField {
             ident: ident.with_span_pos(span),
             expr: e,
             span,
@@ -282,15 +271,18 @@ impl<'a> ExtCtxt<'a> {
         &self,
         span: Span,
         path: ast::Path,
-        fields: Vec<ast::Field>,
+        fields: Vec<ast::ExprField>,
     ) -> P<ast::Expr> {
-        self.expr(span, ast::ExprKind::Struct(path, fields, ast::StructRest::None))
+        self.expr(
+            span,
+            ast::ExprKind::Struct(P(ast::StructExpr { path, fields, rest: ast::StructRest::None })),
+        )
     }
     pub fn expr_struct_ident(
         &self,
         span: Span,
         id: Ident,
-        fields: Vec<ast::Field>,
+        fields: Vec<ast::ExprField>,
     ) -> P<ast::Expr> {
         self.expr_struct(span, self.path_ident(span, id), fields)
     }
@@ -419,7 +411,7 @@ impl<'a> ExtCtxt<'a> {
         &self,
         span: Span,
         path: ast::Path,
-        field_pats: Vec<ast::FieldPat>,
+        field_pats: Vec<ast::PatField>,
     ) -> P<ast::Pat> {
         self.pat(span, PatKind::Struct(path, field_pats, false))
     }
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index 7d0becf1f5d..03c83f9c07b 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -1,12 +1,11 @@
 //! Conditional compilation stripping.
 
-use crate::base::Annotatable;
-
-use rustc_ast::mut_visit::*;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{DelimToken, Token, TokenKind};
-use rustc_ast::tokenstream::{DelimSpan, LazyTokenStream, Spacing, TokenStream, TokenTree};
-use rustc_ast::{self as ast, AstLike, AttrItem, Attribute, MetaItem};
+use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
+use rustc_ast::tokenstream::{DelimSpan, Spacing};
+use rustc_ast::tokenstream::{LazyTokenStream, TokenTree};
+use rustc_ast::{self as ast, AstLike, AttrItem, AttrStyle, Attribute, MetaItem};
 use rustc_attr as attr;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::map_in_place::MapInPlace;
@@ -22,13 +21,14 @@ use rustc_span::edition::{Edition, ALL_EDITIONS};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 
-use smallvec::SmallVec;
-
 /// A folder that strips out items that do not belong in the current configuration.
 pub struct StripUnconfigured<'a> {
     pub sess: &'a Session,
     pub features: Option<&'a Features>,
-    pub modified: bool,
+    /// If `true`, perform cfg-stripping on attached tokens.
+    /// This is only used for the input to derive macros,
+    /// which needs eager expansion of `cfg` and `cfg_attr`
+    pub config_tokens: bool,
 }
 
 fn get_features(
@@ -199,7 +199,7 @@ fn get_features(
 
 // `cfg_attr`-process the crate's attributes and compute the crate's features.
 pub fn features(sess: &Session, mut krate: ast::Crate) -> (ast::Crate, Features) {
-    let mut strip_unconfigured = StripUnconfigured { sess, features: None, modified: false };
+    let mut strip_unconfigured = StripUnconfigured { sess, features: None, config_tokens: false };
 
     let unconfigured_attrs = krate.attrs.clone();
     let diag = &sess.parse_sess.span_diagnostic;
@@ -246,24 +246,83 @@ impl<'a> StripUnconfigured<'a> {
     pub fn configure<T: AstLike>(&mut self, mut node: T) -> Option<T> {
         self.process_cfg_attrs(&mut node);
         if self.in_cfg(node.attrs()) {
+            self.try_configure_tokens(&mut node);
             Some(node)
         } else {
-            self.modified = true;
             None
         }
     }
 
+    fn try_configure_tokens<T: AstLike>(&mut self, node: &mut T) {
+        if self.config_tokens {
+            if let Some(Some(tokens)) = node.tokens_mut() {
+                let attr_annotated_tokens = tokens.create_token_stream();
+                *tokens = LazyTokenStream::new(self.configure_tokens(&attr_annotated_tokens));
+            }
+        }
+    }
+
     fn configure_krate_attrs(
         &mut self,
         mut attrs: Vec<ast::Attribute>,
     ) -> Option<Vec<ast::Attribute>> {
         attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
-        if self.in_cfg(&attrs) {
-            Some(attrs)
-        } else {
-            self.modified = true;
-            None
+        if self.in_cfg(&attrs) { Some(attrs) } else { None }
+    }
+
+    /// Performs cfg-expansion on `stream`, producing a new `AttrAnnotatedTokenStream`.
+    /// This is only used during the invocation of `derive` proc-macros,
+    /// which require that we cfg-expand their entire input.
+    /// Normal cfg-expansion operates on parsed AST nodes via the `configure` method
+    fn configure_tokens(&mut self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream {
+        fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool {
+            stream.0.iter().all(|(tree, _spacing)| match tree {
+                AttrAnnotatedTokenTree::Attributes(_) => false,
+                AttrAnnotatedTokenTree::Token(_) => true,
+                AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner),
+            })
+        }
+
+        if can_skip(stream) {
+            return stream.clone();
         }
+
+        let trees: Vec<_> = stream
+            .0
+            .iter()
+            .flat_map(|(tree, spacing)| match tree.clone() {
+                AttrAnnotatedTokenTree::Attributes(mut data) => {
+                    let mut attrs: Vec<_> = std::mem::take(&mut data.attrs).into();
+                    attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
+                    data.attrs = attrs.into();
+
+                    if self.in_cfg(&data.attrs) {
+                        data.tokens = LazyTokenStream::new(
+                            self.configure_tokens(&data.tokens.create_token_stream()),
+                        );
+                        Some((AttrAnnotatedTokenTree::Attributes(data), *spacing)).into_iter()
+                    } else {
+                        None.into_iter()
+                    }
+                }
+                AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => {
+                    inner = self.configure_tokens(&inner);
+                    Some((AttrAnnotatedTokenTree::Delimited(sp, delim, inner), *spacing))
+                        .into_iter()
+                }
+                AttrAnnotatedTokenTree::Token(token) => {
+                    if let TokenKind::Interpolated(nt) = token.kind {
+                        panic!(
+                            "Nonterminal should have been flattened at {:?}: {:?}",
+                            token.span, nt
+                        );
+                    } else {
+                        Some((AttrAnnotatedTokenTree::Token(token), *spacing)).into_iter()
+                    }
+                }
+            })
+            .collect();
+        AttrAnnotatedTokenStream::new(trees)
     }
 
     /// Parse and expand all `cfg_attr` attributes into a list of attributes
@@ -272,7 +331,7 @@ impl<'a> StripUnconfigured<'a> {
     /// Gives compiler warnings if any `cfg_attr` does not contain any
     /// attributes and is in the original source code. Gives compiler errors if
     /// the syntax of any `cfg_attr` is incorrect.
-    pub fn process_cfg_attrs<T: AstLike>(&mut self, node: &mut T) {
+    fn process_cfg_attrs<T: AstLike>(&mut self, node: &mut T) {
         node.visit_attrs(|attrs| {
             attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
         });
@@ -290,9 +349,6 @@ impl<'a> StripUnconfigured<'a> {
             return vec![attr];
         }
 
-        // A `#[cfg_attr]` either gets removed, or replaced with a new attribute
-        self.modified = true;
-
         let (cfg_predicate, expanded_attrs) = match self.parse_cfg_attr(&attr) {
             None => return vec![],
             Some(r) => r,
@@ -316,7 +372,7 @@ impl<'a> StripUnconfigured<'a> {
         expanded_attrs
             .into_iter()
             .flat_map(|(item, span)| {
-                let orig_tokens = attr.tokens();
+                let orig_tokens = attr.tokens().to_tokenstream();
 
                 // We are taking an attribute of the form `#[cfg_attr(pred, attr)]`
                 // and producing an attribute of the form `#[attr]`. We
@@ -326,25 +382,34 @@ impl<'a> StripUnconfigured<'a> {
 
                 // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
                 // for `attr` when we expand it to `#[attr]`
-                let pound_token = orig_tokens.trees().next().unwrap();
-                if !matches!(pound_token, TokenTree::Token(Token { kind: TokenKind::Pound, .. })) {
-                    panic!("Bad tokens for attribute {:?}", attr);
+                let mut orig_trees = orig_tokens.trees();
+                let pound_token = match orig_trees.next().unwrap() {
+                    TokenTree::Token(token @ Token { kind: TokenKind::Pound, .. }) => token,
+                    _ => panic!("Bad tokens for attribute {:?}", attr),
+                };
+                let pound_span = pound_token.span;
+
+                let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
+                if attr.style == AttrStyle::Inner {
+                    // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
+                    let bang_token = match orig_trees.next().unwrap() {
+                        TokenTree::Token(token @ Token { kind: TokenKind::Not, .. }) => token,
+                        _ => panic!("Bad tokens for attribute {:?}", attr),
+                    };
+                    trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));
                 }
                 // We don't really have a good span to use for the syntheized `[]`
                 // in `#[attr]`, so just use the span of the `#` token.
-                let bracket_group = TokenTree::Delimited(
-                    DelimSpan::from_single(pound_token.span()),
+                let bracket_group = AttrAnnotatedTokenTree::Delimited(
+                    DelimSpan::from_single(pound_span),
                     DelimToken::Bracket,
                     item.tokens
                         .as_ref()
                         .unwrap_or_else(|| panic!("Missing tokens for {:?}", item))
                         .create_token_stream(),
                 );
-                let tokens = Some(LazyTokenStream::new(TokenStream::new(vec![
-                    (pound_token, Spacing::Alone),
-                    (bracket_group, Spacing::Alone),
-                ])));
-
+                trees.push((bracket_group, Spacing::Alone));
+                let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees)));
                 self.process_cfg_attr(attr::mk_attr_from_item(item, tokens, attr.style, span))
             })
             .collect()
@@ -387,7 +452,7 @@ impl<'a> StripUnconfigured<'a> {
     }
 
     /// Determines if a node with the given attributes should be included in this configuration.
-    pub fn in_cfg(&self, attrs: &[Attribute]) -> bool {
+    fn in_cfg(&self, attrs: &[Attribute]) -> bool {
         attrs.iter().all(|attr| {
             if !is_cfg(self.sess, attr) {
                 return true;
@@ -427,16 +492,8 @@ impl<'a> StripUnconfigured<'a> {
         })
     }
 
-    /// Visit attributes on expression and statements (but not attributes on items in blocks).
-    fn visit_expr_attrs(&mut self, attrs: &[Attribute]) {
-        // flag the offending attributes
-        for attr in attrs.iter() {
-            self.maybe_emit_expr_attr_err(attr);
-        }
-    }
-
     /// If attributes are not allowed on expressions, emit an error for `attr`
-    pub fn maybe_emit_expr_attr_err(&self, attr: &Attribute) {
+    crate fn maybe_emit_expr_attr_err(&self, attr: &Attribute) {
         if !self.features.map_or(true, |features| features.stmt_expr_attributes) {
             let mut err = feature_err(
                 &self.sess.parse_sess,
@@ -453,49 +510,10 @@ impl<'a> StripUnconfigured<'a> {
         }
     }
 
-    pub fn configure_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) {
-        let ast::ForeignMod { unsafety: _, abi: _, items } = foreign_mod;
-        items.flat_map_in_place(|item| self.configure(item));
-    }
-
-    fn configure_variant_data(&mut self, vdata: &mut ast::VariantData) {
-        match vdata {
-            ast::VariantData::Struct(fields, ..) | ast::VariantData::Tuple(fields, _) => {
-                fields.flat_map_in_place(|field| self.configure(field))
-            }
-            ast::VariantData::Unit(_) => {}
-        }
-    }
-
-    pub fn configure_item_kind(&mut self, item: &mut ast::ItemKind) {
-        match item {
-            ast::ItemKind::Struct(def, _generics) | ast::ItemKind::Union(def, _generics) => {
-                self.configure_variant_data(def)
-            }
-            ast::ItemKind::Enum(ast::EnumDef { variants }, _generics) => {
-                variants.flat_map_in_place(|variant| self.configure(variant));
-                for variant in variants {
-                    self.configure_variant_data(&mut variant.data);
-                }
-            }
-            _ => {}
-        }
-    }
-
-    pub fn configure_expr_kind(&mut self, expr_kind: &mut ast::ExprKind) {
-        match expr_kind {
-            ast::ExprKind::Match(_m, arms) => {
-                arms.flat_map_in_place(|arm| self.configure(arm));
-            }
-            ast::ExprKind::Struct(_path, fields, _base) => {
-                fields.flat_map_in_place(|field| self.configure(field));
-            }
-            _ => {}
-        }
-    }
-
     pub fn configure_expr(&mut self, expr: &mut P<ast::Expr>) {
-        self.visit_expr_attrs(expr.attrs());
+        for attr in expr.attrs.iter() {
+            self.maybe_emit_expr_attr_err(attr);
+        }
 
         // If an expr is valid to cfg away it will have been removed by the
         // outer stmt or expression folder before descending in here.
@@ -509,118 +527,8 @@ impl<'a> StripUnconfigured<'a> {
             self.sess.parse_sess.span_diagnostic.span_err(attr.span, msg);
         }
 
-        self.process_cfg_attrs(expr)
-    }
-
-    pub fn configure_pat(&mut self, pat: &mut P<ast::Pat>) {
-        if let ast::PatKind::Struct(_path, fields, _etc) = &mut pat.kind {
-            fields.flat_map_in_place(|field| self.configure(field));
-        }
-    }
-
-    pub fn configure_fn_decl(&mut self, fn_decl: &mut ast::FnDecl) {
-        fn_decl.inputs.flat_map_in_place(|arg| self.configure(arg));
-    }
-
-    pub fn fully_configure(&mut self, item: Annotatable) -> Annotatable {
-        // Since the item itself has already been configured by the InvocationCollector,
-        // we know that fold result vector will contain exactly one element
-        match item {
-            Annotatable::Item(item) => Annotatable::Item(self.flat_map_item(item).pop().unwrap()),
-            Annotatable::TraitItem(item) => {
-                Annotatable::TraitItem(self.flat_map_trait_item(item).pop().unwrap())
-            }
-            Annotatable::ImplItem(item) => {
-                Annotatable::ImplItem(self.flat_map_impl_item(item).pop().unwrap())
-            }
-            Annotatable::ForeignItem(item) => {
-                Annotatable::ForeignItem(self.flat_map_foreign_item(item).pop().unwrap())
-            }
-            Annotatable::Stmt(stmt) => {
-                Annotatable::Stmt(stmt.map(|stmt| self.flat_map_stmt(stmt).pop().unwrap()))
-            }
-            Annotatable::Expr(mut expr) => Annotatable::Expr({
-                self.visit_expr(&mut expr);
-                expr
-            }),
-            Annotatable::Arm(arm) => Annotatable::Arm(self.flat_map_arm(arm).pop().unwrap()),
-            Annotatable::Field(field) => {
-                Annotatable::Field(self.flat_map_field(field).pop().unwrap())
-            }
-            Annotatable::FieldPat(fp) => {
-                Annotatable::FieldPat(self.flat_map_field_pattern(fp).pop().unwrap())
-            }
-            Annotatable::GenericParam(param) => {
-                Annotatable::GenericParam(self.flat_map_generic_param(param).pop().unwrap())
-            }
-            Annotatable::Param(param) => {
-                Annotatable::Param(self.flat_map_param(param).pop().unwrap())
-            }
-            Annotatable::StructField(sf) => {
-                Annotatable::StructField(self.flat_map_struct_field(sf).pop().unwrap())
-            }
-            Annotatable::Variant(v) => {
-                Annotatable::Variant(self.flat_map_variant(v).pop().unwrap())
-            }
-        }
-    }
-}
-
-impl<'a> MutVisitor for StripUnconfigured<'a> {
-    fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) {
-        self.configure_foreign_mod(foreign_mod);
-        noop_visit_foreign_mod(foreign_mod, self);
-    }
-
-    fn visit_item_kind(&mut self, item: &mut ast::ItemKind) {
-        self.configure_item_kind(item);
-        noop_visit_item_kind(item, self);
-    }
-
-    fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
-        self.configure_expr(expr);
-        self.configure_expr_kind(&mut expr.kind);
-        noop_visit_expr(expr, self);
-    }
-
-    fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
-        let mut expr = configure!(self, expr);
-        self.configure_expr_kind(&mut expr.kind);
-        noop_visit_expr(&mut expr, self);
-        Some(expr)
-    }
-
-    fn flat_map_generic_param(
-        &mut self,
-        param: ast::GenericParam,
-    ) -> SmallVec<[ast::GenericParam; 1]> {
-        noop_flat_map_generic_param(configure!(self, param), self)
-    }
-
-    fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
-        noop_flat_map_stmt(configure!(self, stmt), self)
-    }
-
-    fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
-        noop_flat_map_item(configure!(self, item), self)
-    }
-
-    fn flat_map_impl_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
-        noop_flat_map_assoc_item(configure!(self, item), self)
-    }
-
-    fn flat_map_trait_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
-        noop_flat_map_assoc_item(configure!(self, item), self)
-    }
-
-    fn visit_pat(&mut self, pat: &mut P<ast::Pat>) {
-        self.configure_pat(pat);
-        noop_visit_pat(pat, self)
-    }
-
-    fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) {
-        self.configure_fn_decl(&mut fn_decl);
-        noop_visit_fn_decl(fn_decl, self);
+        self.process_cfg_attrs(expr);
+        self.try_configure_tokens(&mut *expr);
     }
 }
 
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index b474cad1242..529ef7e4611 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -3,7 +3,7 @@ use crate::config::StripUnconfigured;
 use crate::configure;
 use crate::hygiene::SyntaxContext;
 use crate::mbe::macro_rules::annotate_err_with_kind;
-use crate::module::{parse_external_mod, push_directory, Directory, DirectoryOwnership};
+use crate::module::{mod_dir_path, parse_external_mod, DirOwnership, ParsedExternalMod};
 use crate::placeholders::{placeholder, PlaceholderExpander};
 
 use rustc_ast as ast;
@@ -12,7 +12,7 @@ use rustc_ast::ptr::P;
 use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{self, AssocCtxt, Visitor};
-use rustc_ast::{AstLike, AttrItem, AttrStyle, Block, Inline, ItemKind, LitKind, MacArgs};
+use rustc_ast::{AstLike, AttrItem, Block, Inline, ItemKind, LitKind, MacArgs};
 use rustc_ast::{MacCallStmt, MacStmtStyle, MetaItemKind, ModKind, NestedMetaItem};
 use rustc_ast::{NodeId, PatKind, Path, StmtKind, Unsafe};
 use rustc_ast_pretty::pprust;
@@ -22,7 +22,7 @@ use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, PResult};
 use rustc_feature::Features;
-use rustc_parse::parser::{AttemptLocalParseRecovery, ForceCollect, GateOr, Parser, RecoverComma};
+use rustc_parse::parser::{AttemptLocalParseRecovery, ForceCollect, Parser, RecoverComma};
 use rustc_parse::validate_attr;
 use rustc_session::lint::builtin::UNUSED_DOC_COMMENTS;
 use rustc_session::lint::BuiltinLintDiagnostics;
@@ -177,14 +177,14 @@ ast_fragments! {
     Arms(SmallVec<[ast::Arm; 1]>) {
         "match arm"; many fn flat_map_arm; fn visit_arm(); fn make_arms;
     }
-    Fields(SmallVec<[ast::Field; 1]>) {
-        "field expression"; many fn flat_map_field; fn visit_field(); fn make_fields;
+    Fields(SmallVec<[ast::ExprField; 1]>) {
+        "field expression"; many fn flat_map_expr_field; fn visit_expr_field(); fn make_expr_fields;
     }
-    FieldPats(SmallVec<[ast::FieldPat; 1]>) {
+    FieldPats(SmallVec<[ast::PatField; 1]>) {
         "field pattern";
-        many fn flat_map_field_pattern;
-        fn visit_field_pattern();
-        fn make_field_patterns;
+        many fn flat_map_pat_field;
+        fn visit_pat_field();
+        fn make_pat_fields;
     }
     GenericParams(SmallVec<[ast::GenericParam; 1]>) {
         "generic parameter";
@@ -195,41 +195,47 @@ ast_fragments! {
     Params(SmallVec<[ast::Param; 1]>) {
         "function parameter"; many fn flat_map_param; fn visit_param(); fn make_params;
     }
-    StructFields(SmallVec<[ast::StructField; 1]>) {
+    StructFields(SmallVec<[ast::FieldDef; 1]>) {
         "field";
-        many fn flat_map_struct_field;
-        fn visit_struct_field();
-        fn make_struct_fields;
+        many fn flat_map_field_def;
+        fn visit_field_def();
+        fn make_field_defs;
     }
     Variants(SmallVec<[ast::Variant; 1]>) {
         "variant"; many fn flat_map_variant; fn visit_variant(); fn make_variants;
     }
 }
 
+pub enum SupportsMacroExpansion {
+    No,
+    Yes { supports_inner_attrs: bool },
+}
+
 impl AstFragmentKind {
     crate fn dummy(self, span: Span) -> AstFragment {
         self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment")
     }
 
-    /// Fragment supports macro expansion and not just inert attributes, `cfg` and `cfg_attr`.
-    pub fn supports_macro_expansion(self) -> bool {
+    pub fn supports_macro_expansion(self) -> SupportsMacroExpansion {
         match self {
             AstFragmentKind::OptExpr
             | AstFragmentKind::Expr
-            | AstFragmentKind::Pat
-            | AstFragmentKind::Ty
             | AstFragmentKind::Stmts
-            | AstFragmentKind::Items
+            | AstFragmentKind::Ty
+            | AstFragmentKind::Pat => SupportsMacroExpansion::Yes { supports_inner_attrs: false },
+            AstFragmentKind::Items
             | AstFragmentKind::TraitItems
             | AstFragmentKind::ImplItems
-            | AstFragmentKind::ForeignItems => true,
+            | AstFragmentKind::ForeignItems => {
+                SupportsMacroExpansion::Yes { supports_inner_attrs: true }
+            }
             AstFragmentKind::Arms
             | AstFragmentKind::Fields
             | AstFragmentKind::FieldPats
             | AstFragmentKind::GenericParams
             | AstFragmentKind::Params
             | AstFragmentKind::StructFields
-            | AstFragmentKind::Variants => false,
+            | AstFragmentKind::Variants => SupportsMacroExpansion::No,
         }
     }
 
@@ -243,10 +249,10 @@ impl AstFragmentKind {
                 AstFragment::Arms(items.map(Annotatable::expect_arm).collect())
             }
             AstFragmentKind::Fields => {
-                AstFragment::Fields(items.map(Annotatable::expect_field).collect())
+                AstFragment::Fields(items.map(Annotatable::expect_expr_field).collect())
             }
             AstFragmentKind::FieldPats => {
-                AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect())
+                AstFragment::FieldPats(items.map(Annotatable::expect_pat_field).collect())
             }
             AstFragmentKind::GenericParams => {
                 AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect())
@@ -255,7 +261,7 @@ impl AstFragmentKind {
                 AstFragment::Params(items.map(Annotatable::expect_param).collect())
             }
             AstFragmentKind::StructFields => {
-                AstFragment::StructFields(items.map(Annotatable::expect_struct_field).collect())
+                AstFragment::StructFields(items.map(Annotatable::expect_field_def).collect())
             }
             AstFragmentKind::Variants => {
                 AstFragment::Variants(items.map(Annotatable::expect_variant).collect())
@@ -321,8 +327,8 @@ impl InvocationKind {
         // The assumption is that the attribute expansion cannot change field visibilities,
         // and it holds because only inert attributes are supported in this position.
         match self {
-            InvocationKind::Attr { item: Annotatable::StructField(field), .. }
-            | InvocationKind::Derive { item: Annotatable::StructField(field), .. }
+            InvocationKind::Attr { item: Annotatable::FieldDef(field), .. }
+            | InvocationKind::Derive { item: Annotatable::FieldDef(field), .. }
                 if field.ident.is_none() =>
             {
                 Some(field.vis.clone())
@@ -355,16 +361,17 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     // FIXME: Avoid visiting the crate as a `Mod` item,
     // make crate a first class expansion target instead.
     pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate {
-        let mut module = ModuleData {
-            mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)],
-            directory: match self.cx.source_map().span_to_unmapped_path(krate.span) {
-                FileName::Real(name) => name.into_local_path(),
-                other => PathBuf::from(other.to_string()),
-            },
+        let file_path = match self.cx.source_map().span_to_unmapped_path(krate.span) {
+            FileName::Real(name) => name.into_local_path(),
+            other => PathBuf::from(other.to_string()),
         };
-        module.directory.pop();
-        self.cx.root_path = module.directory.clone();
-        self.cx.current_expansion.module = Rc::new(module);
+        let dir_path = file_path.parent().unwrap_or(&file_path).to_owned();
+        self.cx.root_path = dir_path.clone();
+        self.cx.current_expansion.module = Rc::new(ModuleData {
+            mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)],
+            file_path_stack: vec![file_path],
+            dir_path,
+        });
 
         let krate_item = AstFragment::Items(smallvec![P(ast::Item {
             attrs: krate.attrs,
@@ -484,6 +491,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             let fragment_kind = invoc.fragment_kind;
             let (expanded_fragment, new_invocations) = match self.expand_invoc(invoc, &ext.kind) {
                 ExpandResult::Ready(fragment) => {
+                    let mut derive_invocations = Vec::new();
                     let derive_placeholders = self
                         .cx
                         .resolver
@@ -505,14 +513,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                                 _ => unreachable!(),
                             };
 
-                            invocations.reserve(derives.len());
+                            derive_invocations.reserve(derives.len());
                             derives
                                 .into_iter()
-                                .map(|(_exts, path)| {
+                                .map(|(path, _exts)| {
                                     // FIXME: Consider using the derive resolutions (`_exts`)
                                     // instead of enqueuing the derives to be resolved again later.
                                     let expn_id = ExpnId::fresh(None);
-                                    invocations.push((
+                                    derive_invocations.push((
                                         Invocation {
                                             kind: InvocationKind::Derive {
                                                 path,
@@ -539,7 +547,12 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                         })
                         .unwrap_or_default();
 
-                    self.collect_invocations(fragment, &derive_placeholders)
+                    let (fragment, collected_invocations) =
+                        self.collect_invocations(fragment, &derive_placeholders);
+                    // We choose to expand any derive invocations associated with this macro invocation
+                    // *before* any macro invocations collected from the output fragment
+                    derive_invocations.extend(collected_invocations);
+                    (fragment, derive_invocations)
                 }
                 ExpandResult::Retry(invoc) => {
                     if force {
@@ -598,10 +611,15 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
 
         let invocations = {
             let mut collector = InvocationCollector {
+                // Non-derive macro invocations cannot see the results of cfg expansion - they
+                // will either be removed along with the item, or invoked before the cfg/cfg_attr
+                // attribute is expanded. Therefore, we don't need to configure the tokens
+                // Derive macros *can* see the results of cfg-expansion - they are handled
+                // specially in `fully_expand_fragment`
                 cfg: StripUnconfigured {
                     sess: &self.cx.sess,
                     features: self.cx.ecfg.features,
-                    modified: false,
+                    config_tokens: false,
                 },
                 cx: self.cx,
                 invocations: Vec::new(),
@@ -696,13 +714,26 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 SyntaxExtensionKind::Attr(expander) => {
                     self.gate_proc_macro_input(&item);
                     self.gate_proc_macro_attr_item(span, &item);
-                    let tokens = match attr.style {
-                        AttrStyle::Outer => item.into_tokens(&self.cx.sess.parse_sess),
-                        // FIXME: Properly collect tokens for inner attributes
-                        AttrStyle::Inner => rustc_parse::fake_token_stream(
+                    let mut fake_tokens = false;
+                    if let Annotatable::Item(item_inner) = &item {
+                        if let ItemKind::Mod(_, mod_kind) = &item_inner.kind {
+                            // FIXME: Collect tokens and use them instead of generating
+                            // fake ones. These are unstable, so it needs to be
+                            // fixed prior to stabilization
+                            // Fake tokens when we are invoking an inner attribute, and:
+                            fake_tokens = matches!(attr.style, ast::AttrStyle::Inner) &&
+                                // We are invoking an attribute on the crate root, or an outline
+                                // module
+                                (item_inner.ident.name.is_empty() || !matches!(mod_kind, ast::ModKind::Loaded(_, Inline::Yes, _)));
+                        }
+                    }
+                    let tokens = if fake_tokens {
+                        rustc_parse::fake_token_stream(
                             &self.cx.sess.parse_sess,
                             &item.into_nonterminal(),
-                        ),
+                        )
+                    } else {
+                        item.into_tokens(&self.cx.sess.parse_sess)
                     };
                     let attr_item = attr.unwrap_normal_item();
                     if let MacArgs::Eq(..) = attr_item.args {
@@ -728,7 +759,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                                     });
                                 }
                             };
-                            fragment_kind.expect_from_annotatables(items)
+                            if fragment_kind == AstFragmentKind::Expr && items.is_empty() {
+                                let msg =
+                                    "removing an expression is not supported in this position";
+                                self.cx.span_err(span, msg);
+                                fragment_kind.dummy(span)
+                            } else {
+                                fragment_kind.expect_from_annotatables(items)
+                            }
                         }
                         Err(mut err) => {
                             err.emit();
@@ -786,11 +824,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             }
             Annotatable::Expr(_) => "expressions",
             Annotatable::Arm(..)
-            | Annotatable::Field(..)
-            | Annotatable::FieldPat(..)
+            | Annotatable::ExprField(..)
+            | Annotatable::PatField(..)
             | Annotatable::GenericParam(..)
             | Annotatable::Param(..)
-            | Annotatable::StructField(..)
+            | Annotatable::FieldDef(..)
             | Annotatable::Variant(..) => panic!("unexpected annotatable"),
         };
         if self.cx.ecfg.proc_macro_hygiene() {
@@ -877,21 +915,21 @@ pub fn parse_ast_fragment<'a>(
         }
         AstFragmentKind::TraitItems => {
             let mut items = SmallVec::new();
-            while let Some(item) = this.parse_trait_item()? {
+            while let Some(item) = this.parse_trait_item(ForceCollect::No)? {
                 items.extend(item);
             }
             AstFragment::TraitItems(items)
         }
         AstFragmentKind::ImplItems => {
             let mut items = SmallVec::new();
-            while let Some(item) = this.parse_impl_item()? {
+            while let Some(item) = this.parse_impl_item(ForceCollect::No)? {
                 items.extend(item);
             }
             AstFragment::ImplItems(items)
         }
         AstFragmentKind::ForeignItems => {
             let mut items = SmallVec::new();
-            while let Some(item) = this.parse_foreign_item()? {
+            while let Some(item) = this.parse_foreign_item(ForceCollect::No)? {
                 items.extend(item);
             }
             AstFragment::ForeignItems(items)
@@ -916,7 +954,7 @@ pub fn parse_ast_fragment<'a>(
         }
         AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?),
         AstFragmentKind::Pat => {
-            AstFragment::Pat(this.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::No)?)
+            AstFragment::Pat(this.parse_pat_allow_top_alt(None, RecoverComma::No)?)
         }
         AstFragmentKind::Arms
         | AstFragmentKind::Fields
@@ -1053,13 +1091,23 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
     // since they will not be detected after macro expansion.
     fn check_attributes(&mut self, attrs: &[ast::Attribute]) {
         let features = self.cx.ecfg.features.unwrap();
-        for attr in attrs.iter() {
+        let mut attrs = attrs.iter().peekable();
+        let mut span: Option<Span> = None;
+        while let Some(attr) = attrs.next() {
             rustc_ast_passes::feature_gate::check_attribute(attr, self.cx.sess, features);
             validate_attr::check_meta(&self.cx.sess.parse_sess, attr);
+
+            let current_span = if let Some(sp) = span { sp.to(attr.span) } else { attr.span };
+            span = Some(current_span);
+
+            if attrs.peek().map_or(false, |next_attr| next_attr.doc_str().is_some()) {
+                continue;
+            }
+
             if attr.doc_str().is_some() {
                 self.cx.sess.parse_sess.buffer_lint_with_diagnostic(
                     &UNUSED_DOC_COMMENTS,
-                    attr.span,
+                    current_span,
                     ast::CRATE_NODE_ID,
                     "unused doc comment",
                     BuiltinLintDiagnostics::UnusedDocComment(attr.span),
@@ -1107,28 +1155,28 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
         noop_flat_map_arm(arm, self)
     }
 
-    fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> {
+    fn flat_map_expr_field(&mut self, field: ast::ExprField) -> SmallVec<[ast::ExprField; 1]> {
         let mut field = configure!(self, field);
 
         if let Some(attr) = self.take_first_attr(&mut field) {
             return self
-                .collect_attr(attr, Annotatable::Field(field), AstFragmentKind::Fields)
-                .make_fields();
+                .collect_attr(attr, Annotatable::ExprField(field), AstFragmentKind::Fields)
+                .make_expr_fields();
         }
 
-        noop_flat_map_field(field, self)
+        noop_flat_map_expr_field(field, self)
     }
 
-    fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> {
+    fn flat_map_pat_field(&mut self, fp: ast::PatField) -> SmallVec<[ast::PatField; 1]> {
         let mut fp = configure!(self, fp);
 
         if let Some(attr) = self.take_first_attr(&mut fp) {
             return self
-                .collect_attr(attr, Annotatable::FieldPat(fp), AstFragmentKind::FieldPats)
-                .make_field_patterns();
+                .collect_attr(attr, Annotatable::PatField(fp), AstFragmentKind::FieldPats)
+                .make_pat_fields();
         }
 
-        noop_flat_map_field_pattern(fp, self)
+        noop_flat_map_pat_field(fp, self)
     }
 
     fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> {
@@ -1143,16 +1191,16 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
         noop_flat_map_param(p, self)
     }
 
-    fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> {
+    fn flat_map_field_def(&mut self, sf: ast::FieldDef) -> SmallVec<[ast::FieldDef; 1]> {
         let mut sf = configure!(self, sf);
 
         if let Some(attr) = self.take_first_attr(&mut sf) {
             return self
-                .collect_attr(attr, Annotatable::StructField(sf), AstFragmentKind::StructFields)
-                .make_struct_fields();
+                .collect_attr(attr, Annotatable::FieldDef(sf), AstFragmentKind::StructFields)
+                .make_field_defs();
         }
 
-        noop_flat_map_struct_field(sf, self)
+        noop_flat_map_field_def(sf, self)
     }
 
     fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> {
@@ -1245,10 +1293,12 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
     }
 
     fn visit_block(&mut self, block: &mut P<Block>) {
-        let old_directory_ownership = self.cx.current_expansion.directory_ownership;
-        self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock;
+        let orig_dir_ownership = mem::replace(
+            &mut self.cx.current_expansion.dir_ownership,
+            DirOwnership::UnownedViaBlock,
+        );
         noop_visit_block(block, self);
-        self.cx.current_expansion.directory_ownership = old_directory_ownership;
+        self.cx.current_expansion.dir_ownership = orig_dir_ownership;
     }
 
     fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
@@ -1276,63 +1326,82 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                 })
             }
             ast::ItemKind::Mod(_, ref mut mod_kind) if ident != Ident::invalid() => {
-                let sess = &self.cx.sess.parse_sess;
-                let orig_ownership = self.cx.current_expansion.directory_ownership;
-                let mut module = (*self.cx.current_expansion.module).clone();
-
-                let pushed = &mut false; // Record `parse_external_mod` pushing so we can pop.
-                let dir = Directory { ownership: orig_ownership, path: module.directory };
-                let Directory { ownership, path } = match mod_kind {
-                    ModKind::Loaded(_, Inline::Yes, _) => {
+                let (file_path, dir_path, dir_ownership) = match mod_kind {
+                    ModKind::Loaded(_, inline, _) => {
                         // Inline `mod foo { ... }`, but we still need to push directories.
+                        let (dir_path, dir_ownership) = mod_dir_path(
+                            &self.cx.sess,
+                            ident,
+                            &attrs,
+                            &self.cx.current_expansion.module,
+                            self.cx.current_expansion.dir_ownership,
+                            *inline,
+                        );
                         item.attrs = attrs;
-                        push_directory(&self.cx.sess, ident, &item.attrs, dir)
-                    }
-                    ModKind::Loaded(_, Inline::No, _) => {
-                        panic!("`mod` item is loaded from a file for the second time")
+                        (None, dir_path, dir_ownership)
                     }
                     ModKind::Unloaded => {
                         // We have an outline `mod foo;` so we need to parse the file.
-                        let (items, inner_span, dir) =
-                            parse_external_mod(&self.cx.sess, ident, span, dir, &mut attrs, pushed);
+                        let old_attrs_len = attrs.len();
+                        let ParsedExternalMod {
+                            mut items,
+                            inner_span,
+                            file_path,
+                            dir_path,
+                            dir_ownership,
+                        } = parse_external_mod(
+                            &self.cx.sess,
+                            ident,
+                            span,
+                            &self.cx.current_expansion.module,
+                            self.cx.current_expansion.dir_ownership,
+                            &mut attrs,
+                        );
 
-                        let krate =
-                            ast::Crate { attrs, items, span: inner_span, proc_macros: vec![] };
                         if let Some(extern_mod_loaded) = self.cx.extern_mod_loaded {
-                            extern_mod_loaded(&krate, ident);
+                            (attrs, items) = extern_mod_loaded(ident, attrs, items, inner_span);
                         }
 
-                        *mod_kind = ModKind::Loaded(krate.items, Inline::No, inner_span);
-                        item.attrs = krate.attrs;
-                        // File can have inline attributes, e.g., `#![cfg(...)]` & co. => Reconfigure.
-                        item = match self.configure(item) {
-                            Some(node) => node,
-                            None => {
-                                if *pushed {
-                                    sess.included_mod_stack.borrow_mut().pop();
-                                }
-                                return Default::default();
+                        *mod_kind = ModKind::Loaded(items, Inline::No, inner_span);
+                        item.attrs = attrs;
+                        if item.attrs.len() > old_attrs_len {
+                            // If we loaded an out-of-line module and added some inner attributes,
+                            // then we need to re-configure it and re-collect attributes for
+                            // resolution and expansion.
+                            item = configure!(self, item);
+
+                            if let Some(attr) = self.take_first_attr(&mut item) {
+                                return self
+                                    .collect_attr(
+                                        attr,
+                                        Annotatable::Item(item),
+                                        AstFragmentKind::Items,
+                                    )
+                                    .make_items();
                             }
-                        };
-                        dir
+                        }
+                        (Some(file_path), dir_path, dir_ownership)
                     }
                 };
 
                 // Set the module info before we flat map.
-                self.cx.current_expansion.directory_ownership = ownership;
-                module.directory = path;
+                let mut module = self.cx.current_expansion.module.with_dir_path(dir_path);
                 module.mod_path.push(ident);
+                if let Some(file_path) = file_path {
+                    module.file_path_stack.push(file_path);
+                }
+
                 let orig_module =
                     mem::replace(&mut self.cx.current_expansion.module, Rc::new(module));
+                let orig_dir_ownership =
+                    mem::replace(&mut self.cx.current_expansion.dir_ownership, dir_ownership);
 
                 let result = noop_flat_map_item(item, self);
 
                 // Restore the module info.
+                self.cx.current_expansion.dir_ownership = orig_dir_ownership;
                 self.cx.current_expansion.module = orig_module;
-                self.cx.current_expansion.directory_ownership = orig_ownership;
-                if *pushed {
-                    sess.included_mod_stack.borrow_mut().pop();
-                }
+
                 result
             }
             _ => {
diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs
index c5d8ff25ea9..5fb85867501 100644
--- a/compiler/rustc_expand/src/lib.rs
+++ b/compiler/rustc_expand/src/lib.rs
@@ -1,6 +1,9 @@
+#![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
-#![feature(or_patterns)]
+#![feature(destructuring_assignment)]
+#![feature(iter_zip)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_span)]
diff --git a/compiler/rustc_expand/src/mbe.rs b/compiler/rustc_expand/src/mbe.rs
index cbc4d14a65a..5244ac36bba 100644
--- a/compiler/rustc_expand/src/mbe.rs
+++ b/compiler/rustc_expand/src/mbe.rs
@@ -69,7 +69,7 @@ enum KleeneOp {
     ZeroOrMore,
     /// Kleene plus (`+`) for one or more repetitions
     OneOrMore,
-    /// Kleene optional (`?`) for zero or one reptitions
+    /// Kleene optional (`?`) for zero or one repetitions
     ZeroOrOne,
 }
 
diff --git a/compiler/rustc_expand/src/mbe/macro_check.rs b/compiler/rustc_expand/src/mbe/macro_check.rs
index 91add4f9218..3497e5ad543 100644
--- a/compiler/rustc_expand/src/mbe/macro_check.rs
+++ b/compiler/rustc_expand/src/mbe/macro_check.rs
@@ -116,6 +116,8 @@ use rustc_span::{symbol::MacroRulesNormalizedIdent, MultiSpan, Span};
 
 use smallvec::SmallVec;
 
+use std::iter;
+
 /// Stack represented as linked list.
 ///
 /// Those are used for environments because they grow incrementally and are not mutable.
@@ -204,7 +206,7 @@ pub(super) fn check_meta_variables(
         sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes")
     }
     let mut valid = true;
-    for (lhs, rhs) in lhses.iter().zip(rhses.iter()) {
+    for (lhs, rhs) in iter::zip(lhses, rhses) {
         let mut binders = Binders::default();
         check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid);
         check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid);
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index 73fbde70bda..bc45c57596e 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -18,7 +18,8 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, DiagnosticBuilder};
 use rustc_feature::Features;
-use rustc_lint_defs::builtin::SEMICOLON_IN_EXPRESSIONS_FROM_MACROS;
+use rustc_lint_defs::builtin::{OR_PATTERNS_BACK_COMPAT, SEMICOLON_IN_EXPRESSIONS_FROM_MACROS};
+use rustc_lint_defs::BuiltinLintDiagnostics;
 use rustc_parse::parser::Parser;
 use rustc_session::parse::ParseSess;
 use rustc_session::Session;
@@ -951,8 +952,32 @@ fn check_matcher_core(
         // Now `last` holds the complete set of NT tokens that could
         // end the sequence before SUFFIX. Check that every one works with `suffix`.
         for token in &last.tokens {
-            if let TokenTree::MetaVarDecl(_, name, Some(kind)) = *token {
+            if let TokenTree::MetaVarDecl(span, name, Some(kind)) = *token {
                 for next_token in &suffix_first.tokens {
+                    // Check if the old pat is used and the next token is `|`.
+                    if let NonterminalKind::Pat2015 { inferred: true } = kind {
+                        if let TokenTree::Token(token) = next_token {
+                            if let BinOp(token) = token.kind {
+                                if let token::BinOpToken::Or = token {
+                                    // It is suggestion to use pat2015, for example: $x:pat -> $x:pat2015.
+                                    let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
+                                        span,
+                                        name,
+                                        Some(NonterminalKind::Pat2015 { inferred: false }),
+                                    ));
+                                    sess.buffer_lint_with_diagnostic(
+                                        &OR_PATTERNS_BACK_COMPAT,
+                                        span,
+                                        ast::CRATE_NODE_ID,
+                                        &*format!("the meaning of the `pat` fragment specifier is changing in Rust 2021, which may affect this macro",),
+                                        BuiltinLintDiagnostics::OrPatternsBackCompat(
+                                            span, suggestion,
+                                        ),
+                                    );
+                                }
+                            }
+                        }
+                    }
                     match is_in_follow(next_token, kind) {
                         IsInFollow::Yes => {}
                         IsInFollow::No(possible) => {
@@ -1080,7 +1105,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                     _ => IsInFollow::No(TOKENS),
                 }
             }
-            NonterminalKind::Pat2018 { .. } | NonterminalKind::Pat2021 { .. } => {
+            NonterminalKind::Pat2015 { .. } => {
                 const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
                 match tok {
                     TokenTree::Token(token) => match token.kind {
@@ -1091,6 +1116,17 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                     _ => IsInFollow::No(TOKENS),
                 }
             }
+            NonterminalKind::Pat2021 { .. } => {
+                const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`if`", "`in`"];
+                match tok {
+                    TokenTree::Token(token) => match token.kind {
+                        FatArrow | Comma | Eq => IsInFollow::Yes,
+                        Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
+                        _ => IsInFollow::No(TOKENS),
+                    },
+                    _ => IsInFollow::No(TOKENS),
+                }
+            }
             NonterminalKind::Path | NonterminalKind::Ty => {
                 const TOKENS: &[&str] = &[
                     "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`",
diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs
index c8049495d22..e205cb65d02 100644
--- a/compiler/rustc_expand/src/mbe/quoted.rs
+++ b/compiler/rustc_expand/src/mbe/quoted.rs
@@ -63,13 +63,13 @@ pub(super) fn parse(
                                     let span = token.span.with_lo(start_sp.lo());
 
                                     match frag.name {
-                                        sym::pat2018 | sym::pat2021 => {
+                                        sym::pat2015 | sym::pat2021 => {
                                             if !features.edition_macro_pats {
                                                 feature_err(
                                                     sess,
                                                     sym::edition_macro_pats,
                                                     frag.span,
-                                                    "`pat2018` and `pat2021` are unstable.",
+                                                    "`pat2015` and `pat2021` are unstable.",
                                                 )
                                                 .emit();
                                             }
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index dde65d998d8..f9e7c4254bc 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -209,7 +209,7 @@ pub(super) fn transcribe<'a>(
                             }
                         } else {
                             // 0 is the initial counter (we have done 0 repretitions so far). `len`
-                            // is the total number of reptitions we should generate.
+                            // is the total number of repetitions we should generate.
                             repeats.push((0, len));
 
                             // The first time we encounter the sequence we push it to the stack. It
@@ -362,7 +362,7 @@ impl LockstepIterSize {
 /// appropriate meta-vars in `interpolations`.
 ///
 /// Note that if `repeats` does not match the exact correct depth of a meta-var,
-/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of
+/// `lookup_cur_matched` will return `None`, which is why this still works even in the presence of
 /// multiple nested matcher sequences.
 fn lockstep_iter_size(
     tree: &mbe::TokenTree,
diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs
index 076d3b02be9..c5ce0baaa8f 100644
--- a/compiler/rustc_expand/src/module.rs
+++ b/compiler/rustc_expand/src/module.rs
@@ -1,232 +1,175 @@
+use crate::base::ModuleData;
 use rustc_ast::ptr::P;
-use rustc_ast::{token, Attribute, Item};
-use rustc_errors::{struct_span_err, PResult};
+use rustc_ast::{token, Attribute, Inline, Item};
+use rustc_errors::{struct_span_err, DiagnosticBuilder};
 use rustc_parse::new_parser_from_file;
 use rustc_session::parse::ParseSess;
 use rustc_session::Session;
-use rustc_span::source_map::{FileName, Span};
 use rustc_span::symbol::{sym, Ident};
+use rustc_span::Span;
 
 use std::path::{self, Path, PathBuf};
 
-#[derive(Clone)]
-pub struct Directory {
-    pub path: PathBuf,
-    pub ownership: DirectoryOwnership,
-}
-
 #[derive(Copy, Clone)]
-pub enum DirectoryOwnership {
+pub enum DirOwnership {
     Owned {
         // None if `mod.rs`, `Some("foo")` if we're in `foo.rs`.
         relative: Option<Ident>,
     },
     UnownedViaBlock,
-    UnownedViaMod,
 }
 
-/// Information about the path to a module.
 // Public for rustfmt usage.
-pub struct ModulePath<'a> {
-    name: String,
-    path_exists: bool,
-    pub result: PResult<'a, ModulePathSuccess>,
+pub struct ModulePathSuccess {
+    pub file_path: PathBuf,
+    pub dir_ownership: DirOwnership,
 }
 
-// Public for rustfmt usage.
-pub struct ModulePathSuccess {
-    pub path: PathBuf,
-    pub ownership: DirectoryOwnership,
+crate struct ParsedExternalMod {
+    pub items: Vec<P<Item>>,
+    pub inner_span: Span,
+    pub file_path: PathBuf,
+    pub dir_path: PathBuf,
+    pub dir_ownership: DirOwnership,
+}
+
+pub enum ModError<'a> {
+    CircularInclusion(Vec<PathBuf>),
+    ModInBlock(Option<Ident>),
+    FileNotFound(Ident, PathBuf),
+    MultipleCandidates(Ident, String, String),
+    ParserError(DiagnosticBuilder<'a>),
 }
 
 crate fn parse_external_mod(
     sess: &Session,
-    id: Ident,
+    ident: Ident,
     span: Span, // The span to blame on errors.
-    Directory { mut ownership, path }: Directory,
+    module: &ModuleData,
+    mut dir_ownership: DirOwnership,
     attrs: &mut Vec<Attribute>,
-    pop_mod_stack: &mut bool,
-) -> (Vec<P<Item>>, Span, Directory) {
+) -> ParsedExternalMod {
     // We bail on the first error, but that error does not cause a fatal error... (1)
-    let result: PResult<'_, _> = try {
+    let result: Result<_, ModError<'_>> = try {
         // Extract the file path and the new ownership.
-        let mp = submod_path(sess, id, span, &attrs, ownership, &path)?;
-        ownership = mp.ownership;
+        let mp = mod_file_path(sess, ident, &attrs, &module.dir_path, dir_ownership)?;
+        dir_ownership = mp.dir_ownership;
 
         // Ensure file paths are acyclic.
-        let mut included_mod_stack = sess.parse_sess.included_mod_stack.borrow_mut();
-        error_on_circular_module(&sess.parse_sess, span, &mp.path, &included_mod_stack)?;
-        included_mod_stack.push(mp.path.clone());
-        *pop_mod_stack = true; // We have pushed, so notify caller.
-        drop(included_mod_stack);
+        if let Some(pos) = module.file_path_stack.iter().position(|p| p == &mp.file_path) {
+            Err(ModError::CircularInclusion(module.file_path_stack[pos..].to_vec()))?;
+        }
 
         // Actually parse the external file as a module.
-        let mut parser = new_parser_from_file(&sess.parse_sess, &mp.path, Some(span));
-        let (mut inner_attrs, items, inner_span) = parser.parse_mod(&token::Eof)?;
+        let mut parser = new_parser_from_file(&sess.parse_sess, &mp.file_path, Some(span));
+        let (mut inner_attrs, items, inner_span) =
+            parser.parse_mod(&token::Eof).map_err(|err| ModError::ParserError(err))?;
         attrs.append(&mut inner_attrs);
-        (items, inner_span)
+        (items, inner_span, mp.file_path)
     };
     // (1) ...instead, we return a dummy module.
-    let (items, inner_span) = result.map_err(|mut err| err.emit()).unwrap_or_default();
-
-    // Extract the directory path for submodules of  the module.
-    let path = sess.source_map().span_to_unmapped_path(inner_span);
-    let mut path = match path {
-        FileName::Real(name) => name.into_local_path(),
-        other => PathBuf::from(other.to_string()),
-    };
-    path.pop();
+    let (items, inner_span, file_path) =
+        result.map_err(|err| err.report(sess, span)).unwrap_or_default();
 
-    (items, inner_span, Directory { ownership, path })
-}
+    // Extract the directory path for submodules of the module.
+    let dir_path = file_path.parent().unwrap_or(&file_path).to_owned();
 
-fn error_on_circular_module<'a>(
-    sess: &'a ParseSess,
-    span: Span,
-    path: &Path,
-    included_mod_stack: &[PathBuf],
-) -> PResult<'a, ()> {
-    if let Some(i) = included_mod_stack.iter().position(|p| *p == path) {
-        let mut err = String::from("circular modules: ");
-        for p in &included_mod_stack[i..] {
-            err.push_str(&p.to_string_lossy());
-            err.push_str(" -> ");
-        }
-        err.push_str(&path.to_string_lossy());
-        return Err(sess.span_diagnostic.struct_span_err(span, &err[..]));
-    }
-    Ok(())
+    ParsedExternalMod { items, inner_span, file_path, dir_path, dir_ownership }
 }
 
-crate fn push_directory(
+crate fn mod_dir_path(
     sess: &Session,
-    id: Ident,
+    ident: Ident,
     attrs: &[Attribute],
-    Directory { mut ownership, mut path }: Directory,
-) -> Directory {
-    if let Some(filename) = sess.first_attr_value_str_by_name(attrs, sym::path) {
-        path.push(&*filename.as_str());
-        ownership = DirectoryOwnership::Owned { relative: None };
-    } else {
-        // We have to push on the current module name in the case of relative
-        // paths in order to ensure that any additional module paths from inline
-        // `mod x { ... }` come after the relative extension.
-        //
-        // For example, a `mod z { ... }` inside `x/y.rs` should set the current
-        // directory path to `/x/y/z`, not `/x/z` with a relative offset of `y`.
-        if let DirectoryOwnership::Owned { relative } = &mut ownership {
-            if let Some(ident) = relative.take() {
-                // Remove the relative offset.
-                path.push(&*ident.as_str());
+    module: &ModuleData,
+    mut dir_ownership: DirOwnership,
+    inline: Inline,
+) -> (PathBuf, DirOwnership) {
+    match inline {
+        Inline::Yes => {
+            if let Some(file_path) = mod_file_path_from_attr(sess, attrs, &module.dir_path) {
+                // For inline modules file path from `#[path]` is actually the directory path
+                // for historical reasons, so we don't pop the last segment here.
+                return (file_path, DirOwnership::Owned { relative: None });
             }
-        }
-        path.push(&*id.as_str());
-    }
-    Directory { ownership, path }
-}
 
-fn submod_path<'a>(
-    sess: &'a Session,
-    id: Ident,
-    span: Span,
-    attrs: &[Attribute],
-    ownership: DirectoryOwnership,
-    dir_path: &Path,
-) -> PResult<'a, ModulePathSuccess> {
-    if let Some(path) = submod_path_from_attr(sess, attrs, dir_path) {
-        let ownership = match path.file_name().and_then(|s| s.to_str()) {
-            // All `#[path]` files are treated as though they are a `mod.rs` file.
-            // This means that `mod foo;` declarations inside `#[path]`-included
-            // files are siblings,
+            // We have to push on the current module name in the case of relative
+            // paths in order to ensure that any additional module paths from inline
+            // `mod x { ... }` come after the relative extension.
             //
-            // Note that this will produce weirdness when a file named `foo.rs` is
-            // `#[path]` included and contains a `mod foo;` declaration.
-            // If you encounter this, it's your own darn fault :P
-            Some(_) => DirectoryOwnership::Owned { relative: None },
-            _ => DirectoryOwnership::UnownedViaMod,
-        };
-        return Ok(ModulePathSuccess { ownership, path });
-    }
+            // For example, a `mod z { ... }` inside `x/y.rs` should set the current
+            // directory path to `/x/y/z`, not `/x/z` with a relative offset of `y`.
+            let mut dir_path = module.dir_path.clone();
+            if let DirOwnership::Owned { relative } = &mut dir_ownership {
+                if let Some(ident) = relative.take() {
+                    // Remove the relative offset.
+                    dir_path.push(&*ident.as_str());
+                }
+            }
+            dir_path.push(&*ident.as_str());
 
-    let relative = match ownership {
-        DirectoryOwnership::Owned { relative } => relative,
-        DirectoryOwnership::UnownedViaBlock | DirectoryOwnership::UnownedViaMod => None,
-    };
-    let ModulePath { path_exists, name, result } =
-        default_submod_path(&sess.parse_sess, id, span, relative, dir_path);
-    match ownership {
-        DirectoryOwnership::Owned { .. } => Ok(result?),
-        DirectoryOwnership::UnownedViaBlock => {
-            let _ = result.map_err(|mut err| err.cancel());
-            error_decl_mod_in_block(&sess.parse_sess, span, path_exists, &name)
+            (dir_path, dir_ownership)
         }
-        DirectoryOwnership::UnownedViaMod => {
-            let _ = result.map_err(|mut err| err.cancel());
-            error_cannot_declare_mod_here(&sess.parse_sess, span, path_exists, &name)
+        Inline::No => {
+            // FIXME: This is a subset of `parse_external_mod` without actual parsing,
+            // check whether the logic for unloaded, loaded and inline modules can be unified.
+            let file_path = mod_file_path(sess, ident, &attrs, &module.dir_path, dir_ownership)
+                .map(|mp| {
+                    dir_ownership = mp.dir_ownership;
+                    mp.file_path
+                })
+                .unwrap_or_default();
+
+            // Extract the directory path for submodules of the module.
+            let dir_path = file_path.parent().unwrap_or(&file_path).to_owned();
+
+            (dir_path, dir_ownership)
         }
     }
 }
 
-fn error_decl_mod_in_block<'a, T>(
-    sess: &'a ParseSess,
-    span: Span,
-    path_exists: bool,
-    name: &str,
-) -> PResult<'a, T> {
-    let msg = "Cannot declare a non-inline module inside a block unless it has a path attribute";
-    let mut err = sess.span_diagnostic.struct_span_err(span, msg);
-    if path_exists {
-        let msg = format!("Maybe `use` the module `{}` instead of redeclaring it", name);
-        err.span_note(span, &msg);
+fn mod_file_path<'a>(
+    sess: &'a Session,
+    ident: Ident,
+    attrs: &[Attribute],
+    dir_path: &Path,
+    dir_ownership: DirOwnership,
+) -> Result<ModulePathSuccess, ModError<'a>> {
+    if let Some(file_path) = mod_file_path_from_attr(sess, attrs, dir_path) {
+        // All `#[path]` files are treated as though they are a `mod.rs` file.
+        // This means that `mod foo;` declarations inside `#[path]`-included
+        // files are siblings,
+        //
+        // Note that this will produce weirdness when a file named `foo.rs` is
+        // `#[path]` included and contains a `mod foo;` declaration.
+        // If you encounter this, it's your own darn fault :P
+        let dir_ownership = DirOwnership::Owned { relative: None };
+        return Ok(ModulePathSuccess { file_path, dir_ownership });
     }
-    Err(err)
-}
 
-fn error_cannot_declare_mod_here<'a, T>(
-    sess: &'a ParseSess,
-    span: Span,
-    path_exists: bool,
-    name: &str,
-) -> PResult<'a, T> {
-    let mut err =
-        sess.span_diagnostic.struct_span_err(span, "cannot declare a new module at this location");
-    if !span.is_dummy() {
-        if let FileName::Real(src_name) = sess.source_map().span_to_filename(span) {
-            let src_path = src_name.into_local_path();
-            if let Some(stem) = src_path.file_stem() {
-                let mut dest_path = src_path.clone();
-                dest_path.set_file_name(stem);
-                dest_path.push("mod.rs");
-                err.span_note(
-                    span,
-                    &format!(
-                        "maybe move this module `{}` to its own directory via `{}`",
-                        src_path.display(),
-                        dest_path.display()
-                    ),
-                );
-            }
-        }
-    }
-    if path_exists {
-        err.span_note(
-            span,
-            &format!("... or maybe `use` the module `{}` instead of possibly redeclaring it", name),
-        );
+    let relative = match dir_ownership {
+        DirOwnership::Owned { relative } => relative,
+        DirOwnership::UnownedViaBlock => None,
+    };
+    let result = default_submod_path(&sess.parse_sess, ident, relative, dir_path);
+    match dir_ownership {
+        DirOwnership::Owned { .. } => result,
+        DirOwnership::UnownedViaBlock => Err(ModError::ModInBlock(match result {
+            Ok(_) | Err(ModError::MultipleCandidates(..)) => Some(ident),
+            _ => None,
+        })),
     }
-    Err(err)
 }
 
 /// Derive a submodule path from the first found `#[path = "path_string"]`.
 /// The provided `dir_path` is joined with the `path_string`.
-pub(super) fn submod_path_from_attr(
+fn mod_file_path_from_attr(
     sess: &Session,
     attrs: &[Attribute],
     dir_path: &Path,
 ) -> Option<PathBuf> {
     // Extract path string from first `#[path = "path_string"]` attribute.
-    let path_string = sess.first_attr_value_str_by_name(attrs, sym::path)?;
-    let path_string = path_string.as_str();
+    let path_string = sess.first_attr_value_str_by_name(attrs, sym::path)?.as_str();
 
     // On windows, the base path might have the form
     // `\\?\foo\bar` in which case it does not tolerate
@@ -242,15 +185,14 @@ pub(super) fn submod_path_from_attr(
 // Public for rustfmt usage.
 pub fn default_submod_path<'a>(
     sess: &'a ParseSess,
-    id: Ident,
-    span: Span,
+    ident: Ident,
     relative: Option<Ident>,
     dir_path: &Path,
-) -> ModulePath<'a> {
+) -> Result<ModulePathSuccess, ModError<'a>> {
     // If we're in a foo.rs file instead of a mod.rs file,
     // we need to look for submodules in
-    // `./foo/<id>.rs` and `./foo/<id>/mod.rs` rather than
-    // `./<id>.rs` and `./<id>/mod.rs`.
+    // `./foo/<ident>.rs` and `./foo/<ident>/mod.rs` rather than
+    // `./<ident>.rs` and `./<ident>/mod.rs`.
     let relative_prefix_string;
     let relative_prefix = if let Some(ident) = relative {
         relative_prefix_string = format!("{}{}", ident.name, path::MAIN_SEPARATOR);
@@ -259,7 +201,7 @@ pub fn default_submod_path<'a>(
         ""
     };
 
-    let mod_name = id.name.to_string();
+    let mod_name = ident.name.to_string();
     let default_path_str = format!("{}{}.rs", relative_prefix, mod_name);
     let secondary_path_str =
         format!("{}{}{}mod.rs", relative_prefix, mod_name, path::MAIN_SEPARATOR);
@@ -268,44 +210,74 @@ pub fn default_submod_path<'a>(
     let default_exists = sess.source_map().file_exists(&default_path);
     let secondary_exists = sess.source_map().file_exists(&secondary_path);
 
-    let result = match (default_exists, secondary_exists) {
+    match (default_exists, secondary_exists) {
         (true, false) => Ok(ModulePathSuccess {
-            path: default_path,
-            ownership: DirectoryOwnership::Owned { relative: Some(id) },
+            file_path: default_path,
+            dir_ownership: DirOwnership::Owned { relative: Some(ident) },
         }),
         (false, true) => Ok(ModulePathSuccess {
-            path: secondary_path,
-            ownership: DirectoryOwnership::Owned { relative: None },
+            file_path: secondary_path,
+            dir_ownership: DirOwnership::Owned { relative: None },
         }),
-        (false, false) => {
-            let mut err = struct_span_err!(
-                sess.span_diagnostic,
-                span,
-                E0583,
-                "file not found for module `{}`",
-                mod_name,
-            );
-            err.help(&format!(
-                "to create the module `{}`, create file \"{}\"",
-                mod_name,
-                default_path.display(),
-            ));
-            Err(err)
-        }
+        (false, false) => Err(ModError::FileNotFound(ident, default_path)),
         (true, true) => {
-            let mut err = struct_span_err!(
-                sess.span_diagnostic,
-                span,
-                E0761,
-                "file for module `{}` found at both {} and {}",
-                mod_name,
-                default_path_str,
-                secondary_path_str,
-            );
-            err.help("delete or rename one of them to remove the ambiguity");
-            Err(err)
+            Err(ModError::MultipleCandidates(ident, default_path_str, secondary_path_str))
         }
-    };
+    }
+}
 
-    ModulePath { name: mod_name, path_exists: default_exists || secondary_exists, result }
+impl ModError<'_> {
+    fn report(self, sess: &Session, span: Span) {
+        let diag = &sess.parse_sess.span_diagnostic;
+        match self {
+            ModError::CircularInclusion(file_paths) => {
+                let mut msg = String::from("circular modules: ");
+                for file_path in &file_paths {
+                    msg.push_str(&file_path.display().to_string());
+                    msg.push_str(" -> ");
+                }
+                msg.push_str(&file_paths[0].display().to_string());
+                diag.struct_span_err(span, &msg)
+            }
+            ModError::ModInBlock(ident) => {
+                let msg = "cannot declare a non-inline module inside a block unless it has a path attribute";
+                let mut err = diag.struct_span_err(span, msg);
+                if let Some(ident) = ident {
+                    let note =
+                        format!("maybe `use` the module `{}` instead of redeclaring it", ident);
+                    err.span_note(span, &note);
+                }
+                err
+            }
+            ModError::FileNotFound(ident, default_path) => {
+                let mut err = struct_span_err!(
+                    diag,
+                    span,
+                    E0583,
+                    "file not found for module `{}`",
+                    ident,
+                );
+                err.help(&format!(
+                    "to create the module `{}`, create file \"{}\"",
+                    ident,
+                    default_path.display(),
+                ));
+                err
+            }
+            ModError::MultipleCandidates(ident, default_path_short, secondary_path_short) => {
+                let mut err = struct_span_err!(
+                    diag,
+                    span,
+                    E0761,
+                    "file for module `{}` found at both {} and {}",
+                    ident,
+                    default_path_short,
+                    secondary_path_short,
+                );
+                err.help("delete or rename one of them to remove the ambiguity");
+                err
+            }
+            ModError::ParserError(err) => err,
+        }.emit()
+    }
 }
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
index 98682ba4295..6586ba138fb 100644
--- a/compiler/rustc_expand/src/placeholders.rs
+++ b/compiler/rustc_expand/src/placeholders.rs
@@ -117,7 +117,7 @@ pub fn placeholder(
             span,
             is_placeholder: true,
         }]),
-        AstFragmentKind::Fields => AstFragment::Fields(smallvec![ast::Field {
+        AstFragmentKind::Fields => AstFragment::Fields(smallvec![ast::ExprField {
             attrs: Default::default(),
             expr: expr_placeholder(),
             id,
@@ -126,7 +126,7 @@ pub fn placeholder(
             span,
             is_placeholder: true,
         }]),
-        AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ast::FieldPat {
+        AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ast::PatField {
             attrs: Default::default(),
             id,
             ident,
@@ -153,7 +153,7 @@ pub fn placeholder(
             ty: ty(),
             is_placeholder: true,
         }]),
-        AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ast::StructField {
+        AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ast::FieldDef {
             attrs: Default::default(),
             id,
             ident: None,
@@ -205,19 +205,19 @@ impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> {
         }
     }
 
-    fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> {
+    fn flat_map_expr_field(&mut self, field: ast::ExprField) -> SmallVec<[ast::ExprField; 1]> {
         if field.is_placeholder {
-            self.remove(field.id).make_fields()
+            self.remove(field.id).make_expr_fields()
         } else {
-            noop_flat_map_field(field, self)
+            noop_flat_map_expr_field(field, self)
         }
     }
 
-    fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> {
+    fn flat_map_pat_field(&mut self, fp: ast::PatField) -> SmallVec<[ast::PatField; 1]> {
         if fp.is_placeholder {
-            self.remove(fp.id).make_field_patterns()
+            self.remove(fp.id).make_pat_fields()
         } else {
-            noop_flat_map_field_pattern(fp, self)
+            noop_flat_map_pat_field(fp, self)
         }
     }
 
@@ -240,11 +240,11 @@ impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> {
         }
     }
 
-    fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> {
+    fn flat_map_field_def(&mut self, sf: ast::FieldDef) -> SmallVec<[ast::FieldDef; 1]> {
         if sf.is_placeholder {
-            self.remove(sf.id).make_struct_fields()
+            self.remove(sf.id).make_field_defs()
         } else {
-            noop_flat_map_struct_field(sf, self)
+            noop_flat_map_field_def(sf, self)
         }
     }
 
diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs
index 8cbaa7c945a..3f84979ac05 100644
--- a/compiler/rustc_expand/src/proc_macro.rs
+++ b/compiler/rustc_expand/src/proc_macro.rs
@@ -90,10 +90,11 @@ impl MultiItemModifier for ProcMacroDerive {
             }
             _ => unreachable!(),
         };
-        let input = if item.pretty_printing_compatibility_hack() {
+        let input = if crate::base::pretty_printing_compatibility_hack(&item, &ecx.sess.parse_sess)
+        {
             TokenTree::token(token::Interpolated(Lrc::new(item)), DUMMY_SP).into()
         } else {
-            nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::Yes)
+            nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::No)
         };
 
         let server = proc_macro_server::Rustc::new(ecx);
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index b6195d3bbc4..1ea26b4eab4 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -2,16 +2,21 @@ use crate::base::ExtCtxt;
 
 use rustc_ast as ast;
 use rustc_ast::token;
+use rustc_ast::token::Nonterminal;
+use rustc_ast::token::NtIdent;
 use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens};
 use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::Diagnostic;
+use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
+use rustc_lint_defs::BuiltinLintDiagnostics;
 use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
 use rustc_session::parse::ParseSess;
+use rustc_span::hygiene::ExpnKind;
 use rustc_span::symbol::{self, kw, sym, Symbol};
-use rustc_span::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
+use rustc_span::{BytePos, FileName, MultiSpan, Pos, RealFileName, SourceFile, Span};
 
 use pm::bridge::{server, TokenTree};
 use pm::{Delimiter, Level, LineColumn, Spacing};
@@ -48,11 +53,11 @@ impl ToInternal<token::DelimToken> for Delimiter {
     }
 }
 
-impl FromInternal<(TreeAndSpacing, &'_ ParseSess, &'_ mut Vec<Self>)>
+impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_>)>
     for TokenTree<Group, Punct, Ident, Literal>
 {
     fn from_internal(
-        ((tree, spacing), sess, stack): (TreeAndSpacing, &ParseSess, &mut Vec<Self>),
+        ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec<Self>, &mut Rustc<'_>),
     ) -> Self {
         use rustc_ast::token::*;
 
@@ -141,10 +146,10 @@ impl FromInternal<(TreeAndSpacing, &'_ ParseSess, &'_ mut Vec<Self>)>
             SingleQuote => op!('\''),
 
             Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()),
-            Ident(name, is_raw) => tt!(Ident::new(sess, name, is_raw)),
+            Ident(name, is_raw) => tt!(Ident::new(rustc.sess, name, is_raw)),
             Lifetime(name) => {
                 let ident = symbol::Ident::new(name, span).without_first_quote();
-                stack.push(tt!(Ident::new(sess, ident.name, false)));
+                stack.push(tt!(Ident::new(rustc.sess, ident.name, false)));
                 tt!(Punct::new('\'', true))
             }
             Literal(lit) => tt!(Literal { lit }),
@@ -174,17 +179,15 @@ impl FromInternal<(TreeAndSpacing, &'_ ParseSess, &'_ mut Vec<Self>)>
             }
 
             Interpolated(nt) => {
-                if let Some((name, is_raw)) =
-                    nt.ident_name_compatibility_hack(span, sess.source_map())
-                {
-                    TokenTree::Ident(Ident::new(sess, name.name, is_raw, name.span))
+                if let Some((name, is_raw)) = ident_name_compatibility_hack(&nt, span, rustc) {
+                    TokenTree::Ident(Ident::new(rustc.sess, name.name, is_raw, name.span))
                 } else {
-                    let stream = nt_to_tokenstream(&nt, sess, CanSynthesizeMissingTokens::No);
+                    let stream = nt_to_tokenstream(&nt, rustc.sess, CanSynthesizeMissingTokens::No);
                     TokenTree::Group(Group {
                         delimiter: Delimiter::None,
                         stream,
                         span: DelimSpan::from_single(span),
-                        flatten: nt.pretty_printing_compatibility_hack(),
+                        flatten: crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess),
                     })
                 }
             }
@@ -446,7 +449,7 @@ impl server::TokenStreamIter for Rustc<'_> {
         loop {
             let tree = iter.stack.pop().or_else(|| {
                 let next = iter.cursor.next_with_spacing()?;
-                Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
+                Some(TokenTree::from_internal((next, &mut iter.stack, self)))
             })?;
             // A hack used to pass AST fragments to attribute and derive macros
             // as a single nonterminal token instead of a token stream.
@@ -711,3 +714,100 @@ impl server::Span for Rustc<'_> {
         self.sess.source_map().span_to_snippet(span).ok()
     }
 }
+
+// See issue #74616 for details
+fn ident_name_compatibility_hack(
+    nt: &Nonterminal,
+    orig_span: Span,
+    rustc: &mut Rustc<'_>,
+) -> Option<(rustc_span::symbol::Ident, bool)> {
+    if let NtIdent(ident, is_raw) = nt {
+        if let ExpnKind::Macro(_, macro_name) = orig_span.ctxt().outer_expn_data().kind {
+            let source_map = rustc.sess.source_map();
+            let filename = source_map.span_to_filename(orig_span);
+            if let FileName::Real(RealFileName::Named(path)) = filename {
+                let matches_prefix = |prefix, filename| {
+                    // Check for a path that ends with 'prefix*/src/<filename>'
+                    let mut iter = path.components().rev();
+                    iter.next().and_then(|p| p.as_os_str().to_str()) == Some(filename)
+                        && iter.next().and_then(|p| p.as_os_str().to_str()) == Some("src")
+                        && iter
+                            .next()
+                            .and_then(|p| p.as_os_str().to_str())
+                            .map_or(false, |p| p.starts_with(prefix))
+                };
+
+                let time_macros_impl =
+                    macro_name == sym::impl_macros && matches_prefix("time-macros-impl", "lib.rs");
+                let js_sys = macro_name == sym::arrays && matches_prefix("js-sys", "lib.rs");
+                if time_macros_impl || js_sys {
+                    let snippet = source_map.span_to_snippet(orig_span);
+                    if snippet.as_deref() == Ok("$name") {
+                        if time_macros_impl {
+                            rustc.sess.buffer_lint_with_diagnostic(
+                                &PROC_MACRO_BACK_COMPAT,
+                                orig_span,
+                                ast::CRATE_NODE_ID,
+                                "using an old version of `time-macros-impl`",
+                                BuiltinLintDiagnostics::ProcMacroBackCompat(
+                                "the `time-macros-impl` crate will stop compiling in futures version of Rust. \
+                                Please update to the latest version of the `time` crate to avoid breakage".to_string())
+                            );
+                            return Some((*ident, *is_raw));
+                        }
+                        if js_sys {
+                            if let Some(c) = path
+                                .components()
+                                .flat_map(|c| c.as_os_str().to_str())
+                                .find(|c| c.starts_with("js-sys"))
+                            {
+                                let mut version = c.trim_start_matches("js-sys-").split(".");
+                                if version.next() == Some("0")
+                                    && version.next() == Some("3")
+                                    && version
+                                        .next()
+                                        .and_then(|c| c.parse::<u32>().ok())
+                                        .map_or(false, |v| v < 40)
+                                {
+                                    rustc.sess.buffer_lint_with_diagnostic(
+                                        &PROC_MACRO_BACK_COMPAT,
+                                        orig_span,
+                                        ast::CRATE_NODE_ID,
+                                        "using an old version of `js-sys`",
+                                        BuiltinLintDiagnostics::ProcMacroBackCompat(
+                                        "older versions of the `js-sys` crate will stop compiling in future versions of Rust; \
+                                        please update to `js-sys` v0.3.40 or above".to_string())
+                                    );
+                                    return Some((*ident, *is_raw));
+                                }
+                            }
+                        }
+                    }
+                }
+
+                if macro_name == sym::tuple_from_req && matches_prefix("actix-web", "extract.rs") {
+                    let snippet = source_map.span_to_snippet(orig_span);
+                    if snippet.as_deref() == Ok("$T") {
+                        if let FileName::Real(RealFileName::Named(macro_path)) =
+                            source_map.span_to_filename(rustc.def_site)
+                        {
+                            if macro_path.to_string_lossy().contains("pin-project-internal-0.") {
+                                rustc.sess.buffer_lint_with_diagnostic(
+                                    &PROC_MACRO_BACK_COMPAT,
+                                    orig_span,
+                                    ast::CRATE_NODE_ID,
+                                    "using an old version of `actix-web`",
+                                    BuiltinLintDiagnostics::ProcMacroBackCompat(
+                                    "the version of `actix-web` you are using might stop compiling in future versions of Rust; \
+                                    please update to the latest version of the `actix-web` crate to avoid breakage".to_string())
+                                );
+                                return Some((*ident, *is_raw));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    None
+}
diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs
index 4e818e9feb0..8b546e7e4a3 100644
--- a/compiler/rustc_expand/src/tokenstream/tests.rs
+++ b/compiler/rustc_expand/src/tokenstream/tests.rs
@@ -1,7 +1,7 @@
 use crate::tests::string_to_stream;
 
 use rustc_ast::token;
-use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree};
+use rustc_ast::tokenstream::{Spacing, TokenStream, TokenStreamBuilder, TokenTree};
 use rustc_span::with_default_session_globals;
 use rustc_span::{BytePos, Span, Symbol};
 use smallvec::smallvec;
@@ -14,6 +14,10 @@ fn sp(a: u32, b: u32) -> Span {
     Span::with_root_ctxt(BytePos(a), BytePos(b))
 }
 
+fn joint(tree: TokenTree) -> TokenStream {
+    TokenStream::new(vec![(tree, Spacing::Joint)])
+}
+
 #[test]
 fn test_concat() {
     with_default_session_globals(|| {
@@ -99,8 +103,8 @@ fn test_is_empty() {
 fn test_dotdotdot() {
     with_default_session_globals(|| {
         let mut builder = TokenStreamBuilder::new();
-        builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint());
-        builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint());
+        builder.push(joint(TokenTree::token(token::Dot, sp(0, 1))));
+        builder.push(joint(TokenTree::token(token::Dot, sp(1, 2))));
         builder.push(TokenTree::token(token::Dot, sp(2, 3)));
         let stream = builder.build();
         assert!(stream.eq_unspanned(&string_to_ts("...")));
diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs
index 5012d500cee..e8642a52749 100644
--- a/compiler/rustc_feature/src/accepted.rs
+++ b/compiler/rustc_feature/src/accepted.rs
@@ -275,6 +275,12 @@ declare_features! (
     (accepted, move_ref_pattern, "1.49.0", Some(68354), None),
     /// The smallest useful subset of `const_generics`.
     (accepted, min_const_generics, "1.51.0", Some(74878), None),
+    /// The `unsafe_op_in_unsafe_fn` lint (allowed by default): no longer treat an unsafe function as an unsafe block.
+    (accepted, unsafe_block_in_unsafe_fn, "1.52.0", Some(71668), None),
+    /// Allows the use of or-patterns (e.g., `0 | 1`).
+    (accepted, or_patterns, "1.53.0", Some(54883), None),
+    /// Allows defining identifiers beyond ASCII.
+    (accepted, non_ascii_idents, "1.53.0", Some(55467), None),
 
     // -------------------------------------------------------------------------
     // feature-group-end: accepted features
diff --git a/compiler/rustc_feature/src/active.rs b/compiler/rustc_feature/src/active.rs
index 8ee995a59d8..a410826d3fd 100644
--- a/compiler/rustc_feature/src/active.rs
+++ b/compiler/rustc_feature/src/active.rs
@@ -134,9 +134,6 @@ declare_features! (
     /// Allows using the `box $expr` syntax.
     (active, box_syntax, "1.0.0", Some(49733), None),
 
-    /// Allows using `#[main]` to replace the entrypoint `#[lang = "start"]` calls.
-    (active, main, "1.0.0", Some(29634), None),
-
     /// Allows using `#[start]` on a function indicating that it is the program entrypoint.
     (active, start, "1.0.0", Some(29633), None),
 
@@ -216,6 +213,10 @@ declare_features! (
     /// Renamed from `optin_builtin_traits`.
     (active, auto_traits, "1.50.0", Some(13231), None),
 
+    /// Allows `#[doc(notable_trait)]`.
+    /// Renamed from `doc_spotlight`.
+    (active, doc_notable_trait, "1.52.0", Some(45040), None),
+
     // no-tracking-issue-end
 
     // -------------------------------------------------------------------------
@@ -254,12 +255,6 @@ declare_features! (
     // feature-group-start: actual feature gates
     // -------------------------------------------------------------------------
 
-    /// Allows using the `#[link_args]` attribute.
-    (active, link_args, "1.0.0", Some(29596), None),
-
-    /// Allows defining identifiers beyond ASCII.
-    (active, non_ascii_idents, "1.0.0", Some(55467), None),
-
     /// Allows using `#[plugin_registrar]` on functions.
     (active, plugin_registrar, "1.0.0", Some(29597), None),
 
@@ -374,9 +369,6 @@ declare_features! (
     /// Allows `#[doc(masked)]`.
     (active, doc_masked, "1.21.0", Some(44027), None),
 
-    /// Allows `#[doc(spotlight)]`.
-    (active, doc_spotlight, "1.22.0", Some(45040), None),
-
     /// Allows `#[doc(include = "some-file")]`.
     (active, external_doc, "1.22.0", Some(44732), None),
 
@@ -488,9 +480,6 @@ declare_features! (
     /// Allows `impl Trait` to be used inside type aliases (RFC 2515).
     (active, type_alias_impl_trait, "1.38.0", Some(63063), None),
 
-    /// Allows the use of or-patterns (e.g., `0 | 1`).
-    (active, or_patterns, "1.38.0", Some(54883), None),
-
     /// Allows the definition of `const extern fn` and `const unsafe extern fn`.
     (active, const_extern_fn, "1.40.0", Some(64926), None),
 
@@ -557,9 +546,6 @@ declare_features! (
     /// Allows the use of `#[ffi_const]` on foreign functions.
     (active, ffi_const, "1.45.0", Some(58328), None),
 
-    /// No longer treat an unsafe function as an unsafe block.
-    (active, unsafe_block_in_unsafe_fn, "1.45.0", Some(71668), None),
-
     /// Allows `extern "avr-interrupt" fn()` and `extern "avr-non-blocking-interrupt" fn()`.
     (active, abi_avr_interrupt, "1.45.0", Some(69664), None),
 
@@ -617,7 +603,7 @@ declare_features! (
     /// Allows arbitrary expressions in key-value attributes at parse time.
     (active, extended_key_value_attributes, "1.50.0", Some(78835), None),
 
-    /// `:pat2018` and `:pat2021` macro matchers.
+    /// `:pat2015` and `:pat2021` macro matchers.
     (active, edition_macro_pats, "1.51.0", Some(54883), None),
 
     /// Allows const generics to have default values (e.g. `struct Foo<const N: usize = 3>(...);`).
@@ -641,9 +627,21 @@ declare_features! (
     /// Allows `pub` on `macro_rules` items.
     (active, pub_macro_rules, "1.52.0", Some(78855), None),
 
+    /// Allows the use of type alias impl trait in function return positions
+    (active, min_type_alias_impl_trait, "1.52.0", Some(63063), None),
+
     /// Allows associated types in inherent impls.
     (active, inherent_associated_types, "1.52.0", Some(8995), None),
 
+    /// Allows `extern "C-unwind" fn` to enable unwinding across ABI boundaries.
+    (active, c_unwind, "1.52.0", Some(74990), None),
+
+    /// Allows using `#[repr(align(...))]` on function items
+    (active, fn_align, "1.53.0", Some(82232), None),
+
+    /// Allows `extern "wasm" fn`
+    (active, wasm_abi, "1.53.0", Some(83788), None),
+
     // -------------------------------------------------------------------------
     // feature-group-end: actual feature gates
     // -------------------------------------------------------------------------
@@ -670,6 +668,7 @@ pub const INCOMPLETE_FEATURES: &[Symbol] = &[
     sym::capture_disjoint_fields,
     sym::const_generics_defaults,
     sym::inherent_associated_types,
+    sym::type_alias_impl_trait,
 ];
 
 /// Some features are not allowed to be used together at the same time, if
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 072062dd615..43054f5bf5e 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -227,8 +227,8 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         template!(List: r#"name = "...", /*opt*/ kind = "dylib|static|...", /*opt*/ wasm_import_module = "...""#),
     ),
     ungated!(link_name, AssumedUsed, template!(NameValueStr: "name")),
-    ungated!(no_link, Normal, template!(Word)),
-    ungated!(repr, Normal, template!(List: "C")),
+    ungated!(no_link, AssumedUsed, template!(Word)),
+    ungated!(repr, AssumedUsed, template!(List: "C")),
     ungated!(export_name, AssumedUsed, template!(NameValueStr: "name")),
     ungated!(link_section, AssumedUsed, template!(NameValueStr: "name")),
     ungated!(no_mangle, AssumedUsed, template!(Word)),
@@ -280,11 +280,6 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     // Linking:
     gated!(naked, AssumedUsed, template!(Word), naked_functions, experimental!(naked)),
     gated!(
-        link_args, Normal, template!(NameValueStr: "args"),
-        "the `link_args` attribute is experimental and not portable across platforms, \
-        it is recommended to use `#[link(name = \"foo\")] instead",
-    ),
-    gated!(
         link_ordinal, AssumedUsed, template!(List: "ordinal"), raw_dylib,
         experimental!(link_ordinal)
     ),
@@ -322,7 +317,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         "custom test frameworks are an unstable feature",
     ),
     // RFC #1268
-    gated!(marker, Normal, template!(Word), marker_trait_attr, experimental!(marker)),
+    gated!(marker, AssumedUsed, template!(Word), marker_trait_attr, experimental!(marker)),
     gated!(
         thread_local, AssumedUsed, template!(Word),
         "`#[thread_local]` is an experimental feature, and does not currently handle destructors",
@@ -541,6 +536,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_specialization_trait, Normal, template!(Word),
         "the `#[rustc_specialization_trait]` attribute is used to check specializations"
     ),
+    rustc_attr!(
+        rustc_main, Normal, template!(Word),
+        "the `#[rustc_main]` attribute is used internally to specify test entry point function",
+    ),
 
     // ==========================================================================
     // Internal attributes, Testing:
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index 2a7c2a02fba..654d2408580 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -1,7 +1,7 @@
 //! # Feature gates
 //!
 //! This crate declares the set of past and present unstable features in the compiler.
-//! Feature gate checking itself is done in `librustc_ast_passes/feature_gate.rs`
+//! Feature gate checking itself is done in `rustc_ast_passes/src/feature_gate.rs`
 //! at the moment.
 //!
 //! Features are enabled in programs via the crate-level attributes of
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 38a3a4e3d44..c2ad339ed41 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -80,6 +80,11 @@ declare_features! (
      Some("subsumed by `#![feature(allocator_internals)]`")),
     /// Allows identifying crates that contain sanitizer runtimes.
     (removed, sanitizer_runtime, "1.17.0", None, None, None),
+    /// Allows `#[doc(spotlight)]`.
+    /// The attribute was renamed to `#[doc(notable_trait)]`
+    /// and the feature to `doc_notable_trait`.
+    (removed, doc_spotlight, "1.22.0", Some(45040), None,
+     Some("renamed to `doc_notable_trait`")),
     (removed, proc_macro_mod, "1.27.0", Some(54727), None,
      Some("subsumed by `#![feature(proc_macro_hygiene)]`")),
     (removed, proc_macro_expr, "1.27.0", Some(54727), None,
@@ -106,7 +111,7 @@ declare_features! (
      Some("subsumed by `.await` syntax")),
     /// Allows defining `existential type`s.
     (removed, existential_type, "1.38.0", Some(63063), None,
-     Some("removed in favor of `#![feature(type_alias_impl_trait)]`")),
+     Some("removed in favor of `#![feature(min_type_alias_impl_trait)]`")),
     /// Allows using the macros:
     /// + `__diagnostic_used`
     /// + `__register_diagnostic`
@@ -123,6 +128,12 @@ declare_features! (
     /// Allows comparing raw pointers during const eval.
     (removed, const_compare_raw_pointers, "1.46.0", Some(53020), None,
      Some("cannot be allowed in const eval in any meaningful way")),
+    /// Allows using the `#[link_args]` attribute.
+    (removed, link_args, "1.53.0", Some(29596), None,
+     Some("removed in favor of using `-C link-arg=ARG` on command line, \
+           which is available from cargo build scripts with `cargo:rustc-link-arg` now")),
+    /// Allows using `#[main]` to replace the entrypoint `#[lang = "start"]` calls.
+    (removed, main, "1.53.0", Some(29634), None, None),
 
     // -------------------------------------------------------------------------
     // feature-group-end: removed features
diff --git a/compiler/rustc_graphviz/src/lib.rs b/compiler/rustc_graphviz/src/lib.rs
index 9653ff022f1..db70beb5914 100644
--- a/compiler/rustc_graphviz/src/lib.rs
+++ b/compiler/rustc_graphviz/src/lib.rs
@@ -413,10 +413,6 @@ impl<'a> Id<'a> {
     pub fn as_slice(&'a self) -> &'a str {
         &*self.name
     }
-
-    pub fn name(self) -> Cow<'a, str> {
-        self.name
-    }
 }
 
 /// Each instance of a type that implements `Label<C>` maps to a
@@ -484,10 +480,6 @@ impl<'a> LabelText<'a> {
         LabelStr(s.into())
     }
 
-    pub fn escaped<S: Into<Cow<'a, str>>>(s: S) -> LabelText<'a> {
-        EscStr(s.into())
-    }
-
     pub fn html<S: Into<Cow<'a, str>>>(s: S) -> LabelText<'a> {
         HtmlStr(s.into())
     }
@@ -543,11 +535,6 @@ impl<'a> LabelText<'a> {
         }
     }
 
-    /// Puts `prefix` on a line above this label, with a blank line separator.
-    pub fn prefix_line(self, prefix: LabelText<'_>) -> LabelText<'static> {
-        prefix.suffix_line(self)
-    }
-
     /// Puts `suffix` on a line below this label, with a blank line separator.
     pub fn suffix_line(self, suffix: LabelText<'_>) -> LabelText<'static> {
         let mut prefix = self.pre_escaped_content().into_owned();
@@ -602,11 +589,6 @@ pub enum RenderOption {
     DarkTheme,
 }
 
-/// Returns vec holding all the default render options.
-pub fn default_options() -> Vec<RenderOption> {
-    vec![]
-}
-
 /// Renders directed graph `g` into the writer `w` in DOT syntax.
 /// (Simple wrapper around `render_opts` that passes a default set of options.)
 pub fn render<'a, N, E, G, W>(g: &'a G, w: &mut W) -> io::Result<()>
diff --git a/compiler/rustc_graphviz/src/tests.rs b/compiler/rustc_graphviz/src/tests.rs
index 70b8197f5ef..a297bac86c4 100644
--- a/compiler/rustc_graphviz/src/tests.rs
+++ b/compiler/rustc_graphviz/src/tests.rs
@@ -111,7 +111,7 @@ impl<'a> Labeller<'a> for LabelledGraph {
     fn node_label(&'a self, n: &Node) -> LabelText<'a> {
         match self.node_labels[*n] {
             Some(l) => LabelStr(l.into()),
-            None => LabelStr(id_name(n).name()),
+            None => LabelStr(id_name(n).name),
         }
     }
     fn edge_label(&'a self, e: &&'a Edge) -> LabelText<'a> {
diff --git a/compiler/rustc_hir/src/arena.rs b/compiler/rustc_hir/src/arena.rs
index c7dc66b70fe..ddf82186169 100644
--- a/compiler/rustc_hir/src/arena.rs
+++ b/compiler/rustc_hir/src/arena.rs
@@ -25,8 +25,8 @@ macro_rules! arena_types {
             [] generic_bound: rustc_hir::GenericBound<$tcx>,
             [] generic_param: rustc_hir::GenericParam<$tcx>,
             [] expr: rustc_hir::Expr<$tcx>,
-            [] field: rustc_hir::Field<$tcx>,
-            [] field_pat: rustc_hir::FieldPat<$tcx>,
+            [] expr_field: rustc_hir::ExprField<$tcx>,
+            [] pat_field: rustc_hir::PatField<$tcx>,
             [] fn_decl: rustc_hir::FnDecl<$tcx>,
             [] foreign_item: rustc_hir::ForeignItem<$tcx>,
             [few] foreign_item_ref: rustc_hir::ForeignItemRef<$tcx>,
@@ -42,7 +42,7 @@ macro_rules! arena_types {
             [] poly_trait_ref: rustc_hir::PolyTraitRef<$tcx>,
             [] qpath: rustc_hir::QPath<$tcx>,
             [] stmt: rustc_hir::Stmt<$tcx>,
-            [] struct_field: rustc_hir::StructField<$tcx>,
+            [] field_def: rustc_hir::FieldDef<$tcx>,
             [] trait_item_ref: rustc_hir::TraitItemRef,
             [] ty: rustc_hir::Ty<$tcx>,
             [] type_binding: rustc_hir::TypeBinding<$tcx>,
diff --git a/compiler/rustc_hir/src/def.rs b/compiler/rustc_hir/src/def.rs
index a81eb747a33..de10d88c1d2 100644
--- a/compiler/rustc_hir/src/def.rs
+++ b/compiler/rustc_hir/src/def.rs
@@ -20,6 +20,7 @@ pub enum CtorOf {
     Variant,
 }
 
+/// What kind of constructor something is.
 #[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
 #[derive(HashStable_Generic)]
 pub enum CtorKind {
@@ -31,6 +32,7 @@ pub enum CtorKind {
     Fictive,
 }
 
+/// An attribute that is not a macro; e.g., `#[inline]` or `#[rustfmt::skip]`.
 #[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
 #[derive(HashStable_Generic)]
 pub enum NonMacroAttrKind {
@@ -47,33 +49,51 @@ pub enum NonMacroAttrKind {
     Registered,
 }
 
+/// What kind of definition something is; e.g., `mod` vs `struct`.
 #[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
 #[derive(HashStable_Generic)]
 pub enum DefKind {
     // Type namespace
     Mod,
-    /// Refers to the struct itself, `DefKind::Ctor` refers to its constructor if it exists.
+    /// Refers to the struct itself, [`DefKind::Ctor`] refers to its constructor if it exists.
     Struct,
     Union,
     Enum,
-    /// Refers to the variant itself, `DefKind::Ctor` refers to its constructor if it exists.
+    /// Refers to the variant itself, [`DefKind::Ctor`] refers to its constructor if it exists.
     Variant,
     Trait,
-    /// `type Foo = Bar;`
+    /// Type alias: `type Foo = Bar;`
     TyAlias,
+    /// Type from an `extern` block.
     ForeignTy,
+    /// Trait alias: `trait IntIterator = Iterator<Item = i32>;`
     TraitAlias,
+    /// Associated type: `trait MyTrait { type Assoc; }`
     AssocTy,
+    /// Type parameter: the `T` in `struct Vec<T> { ... }`
     TyParam,
 
     // Value namespace
     Fn,
     Const,
+    /// Constant generic parameter: `struct Foo<const N: usize> { ... }`
     ConstParam,
     Static,
     /// Refers to the struct or enum variant's constructor.
+    ///
+    /// The reason `Ctor` exists in addition to [`DefKind::Struct`] and
+    /// [`DefKind::Variant`] is because structs and enum variants exist
+    /// in the *type* namespace, whereas struct and enum variant *constructors*
+    /// exist in the *value* namespace.
+    ///
+    /// You may wonder why enum variants exist in the type namespace as opposed
+    /// to the value namespace. Check out [RFC 2593] for intuition on why that is.
+    ///
+    /// [RFC 2593]: https://github.com/rust-lang/rfcs/pull/2593
     Ctor(CtorOf, CtorKind),
+    /// Associated function: `impl MyStruct { fn associated() {} }`
     AssocFn,
+    /// Associated constant: `trait MyTrait { const ASSOC: usize; }`
     AssocConst,
 
     // Macro namespace
@@ -82,11 +102,16 @@ pub enum DefKind {
     // Not namespaced (or they are, but we don't treat them so)
     ExternCrate,
     Use,
+    /// An `extern` block.
     ForeignMod,
+    /// Anonymous constant, e.g. the `1 + 2` in `[u8; 1 + 2]`, or `const { 1 + 2}`
     AnonConst,
+    /// Opaque type, aka `impl Trait`.
     OpaqueTy,
     Field,
+    /// Lifetime parameter: the `'a` in `struct Foo<'a> { ... }`
     LifetimeParam,
+    /// A use of [`global_asm!`].
     GlobalAsm,
     Impl,
     Closure,
@@ -196,35 +221,130 @@ impl DefKind {
 }
 
 /// The resolution of a path or export.
+///
+/// For every path or identifier in Rust, the compiler must determine
+/// what the path refers to. This process is called name resolution,
+/// and `Res` is the primary result of name resolution.
+///
+/// For example, everything prefixed with `/* Res */` in this example has
+/// an associated `Res`:
+///
+/// ```
+/// fn str_to_string(s: & /* Res */ str) -> /* Res */ String {
+///     /* Res */ String::from(/* Res */ s)
+/// }
+///
+/// /* Res */ str_to_string("hello");
+/// ```
+///
+/// The associated `Res`s will be:
+///
+/// - `str` will resolve to [`Res::PrimTy`];
+/// - `String` will resolve to [`Res::Def`], and the `Res` will include the [`DefId`]
+///   for `String` as defined in the standard library;
+/// - `String::from` will also resolve to [`Res::Def`], with the [`DefId`]
+///   pointing to `String::from`;
+/// - `s` will resolve to [`Res::Local`];
+/// - the call to `str_to_string` will resolve to [`Res::Def`], with the [`DefId`]
+///   pointing to the definition of `str_to_string` in the current crate.
+//
 #[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
 #[derive(HashStable_Generic)]
 pub enum Res<Id = hir::HirId> {
+    /// Definition having a unique ID (`DefId`), corresponds to something defined in user code.
+    ///
+    /// **Not bound to a specific namespace.**
     Def(DefKind, DefId),
 
     // Type namespace
+    /// A primitive type such as `i32` or `str`.
+    ///
+    /// **Belongs to the type namespace.**
     PrimTy(hir::PrimTy),
-    /// `Self`, with both an optional trait and impl `DefId`.
+    /// The `Self` type, optionally with the trait it is associated with
+    /// and optionally with the [`DefId`] of the impl it is associated with.
+    ///
+    /// **Belongs to the type namespace.**
+    ///
+    /// For example, the `Self` in
     ///
-    /// HACK(min_const_generics): impl self types also have an optional requirement to not mention
+    /// ```
+    /// trait Foo {
+    ///     fn foo() -> Box<Self>;
+    /// }
+    /// ```
+    ///
+    /// would have the [`DefId`] of `Foo` associated with it. The `Self` in
+    ///
+    /// ```
+    /// struct Bar;
+    ///
+    /// impl Bar {
+    ///     fn new() -> Self { Bar }
+    /// }
+    /// ```
+    ///
+    /// would have the [`DefId`] of the impl associated with it. Finally, the `Self` in
+    ///
+    /// ```
+    /// impl Foo for Bar {
+    ///     fn foo() -> Box<Self> { Box::new(Bar) }
+    /// }
+    /// ```
+    ///
+    /// would have both the [`DefId`] of `Foo` and the [`DefId`] of the impl
+    /// associated with it.
+    ///
+    /// *See also [`Res::SelfCtor`].*
+    ///
+    /// -----
+    ///
+    /// HACK(min_const_generics): impl self types also have an optional requirement to **not** mention
     /// any generic parameters to allow the following with `min_const_generics`:
-    /// ```rust
-    /// impl Foo { fn test() -> [u8; std::mem::size_of::<Self>()] {} }
+    /// ```
+    /// impl Foo { fn test() -> [u8; std::mem::size_of::<Self>()] { todo!() } }
     /// ```
     /// We do however allow `Self` in repeat expression even if it is generic to not break code
     /// which already works on stable while causing the `const_evaluatable_unchecked` future compat lint.
     ///
     /// FIXME(lazy_normalization_consts): Remove this bodge once that feature is stable.
-    SelfTy(Option<DefId> /* trait */, Option<(DefId, bool)> /* impl */),
-    ToolMod, // e.g., `rustfmt` in `#[rustfmt::skip]`
+    SelfTy(
+        /// Optionally, the trait associated with this `Self` type.
+        Option<DefId>,
+        /// Optionally, the impl associated with this `Self` type.
+        Option<(DefId, bool)>,
+    ),
+    /// A tool attribute module; e.g., the `rustfmt` in `#[rustfmt::skip]`.
+    ///
+    /// **Belongs to the type namespace.**
+    ToolMod,
 
     // Value namespace
-    SelfCtor(DefId /* impl */), // `DefId` refers to the impl
+    /// The `Self` constructor, along with the [`DefId`]
+    /// of the impl it is associated with.
+    ///
+    /// **Belongs to the value namespace.**
+    ///
+    /// *See also [`Res::SelfTy`].*
+    SelfCtor(DefId),
+    /// A local variable or function parameter.
+    ///
+    /// **Belongs to the value namespace.**
     Local(Id),
 
     // Macro namespace
+    /// An attribute that is *not* implemented via macro.
+    /// E.g., `#[inline]` and `#[rustfmt::skip]`, which are essentially directives,
+    /// as opposed to `#[test]`, which is a builtin macro.
+    ///
+    /// **Belongs to the macro namespace.**
     NonMacroAttr(NonMacroAttrKind), // e.g., `#[inline]` or `#[rustfmt::skip]`
 
     // All namespaces
+    /// Name resolution failed. We use a dummy `Res` variant so later phases
+    /// of the compiler won't crash and can instead report more errors.
+    ///
+    /// **Not bound to a specific namespace.**
     Err,
 }
 
@@ -275,17 +395,26 @@ impl PartialRes {
     }
 }
 
-/// Different kinds of symbols don't influence each other.
-///
-/// Therefore, they have a separate universe (namespace).
+/// Different kinds of symbols can coexist even if they share the same textual name.
+/// Therefore, they each have a separate universe (known as a "namespace").
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
 pub enum Namespace {
+    /// The type namespace includes `struct`s, `enum`s, `union`s, `trait`s, and `mod`s
+    /// (and, by extension, crates).
+    ///
+    /// Note that the type namespace includes other items; this is not an
+    /// exhaustive list.
     TypeNS,
+    /// The value namespace includes `fn`s, `const`s, `static`s, and local variables (including function arguments).
     ValueNS,
+    /// The macro namespace includes `macro_rules!` macros, declarative `macro`s,
+    /// procedural macros, attribute macros, `derive` macros, and non-macro attributes
+    /// like `#[inline]` and `#[rustfmt::skip]`.
     MacroNS,
 }
 
 impl Namespace {
+    /// The English description of the namespace.
     pub fn descr(self) -> &'static str {
         match self {
             Self::TypeNS => "type",
diff --git a/compiler/rustc_hir/src/definitions.rs b/compiler/rustc_hir/src/definitions.rs
index 6a1b9bdbb94..0f77de9fb25 100644
--- a/compiler/rustc_hir/src/definitions.rs
+++ b/compiler/rustc_hir/src/definitions.rs
@@ -5,13 +5,16 @@
 //! expressions) that are mostly just leftovers.
 
 pub use crate::def_id::DefPathHash;
-use crate::def_id::{CrateNum, DefId, DefIndex, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE};
+use crate::def_id::{
+    CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, CRATE_DEF_INDEX, LOCAL_CRATE,
+};
 use crate::hir;
 
-use rustc_ast::crate_disambiguator::CrateDisambiguator;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::StableHasher;
+use rustc_data_structures::unhash::UnhashMap;
 use rustc_index::vec::IndexVec;
+use rustc_span::crate_disambiguator::CrateDisambiguator;
 use rustc_span::hygiene::ExpnId;
 use rustc_span::symbol::{kw, sym, Symbol};
 
@@ -27,6 +30,7 @@ use tracing::debug;
 pub struct DefPathTable {
     index_to_key: IndexVec<DefIndex, DefKey>,
     def_path_hashes: IndexVec<DefIndex, DefPathHash>,
+    def_path_hash_to_index: UnhashMap<DefPathHash, DefIndex>,
 }
 
 impl DefPathTable {
@@ -39,6 +43,35 @@ impl DefPathTable {
         };
         self.def_path_hashes.push(def_path_hash);
         debug_assert!(self.def_path_hashes.len() == self.index_to_key.len());
+
+        // Check for hash collisions of DefPathHashes. These should be
+        // exceedingly rare.
+        if let Some(existing) = self.def_path_hash_to_index.insert(def_path_hash, index) {
+            let def_path1 = DefPath::make(LOCAL_CRATE, existing, |idx| self.def_key(idx));
+            let def_path2 = DefPath::make(LOCAL_CRATE, index, |idx| self.def_key(idx));
+
+            // Continuing with colliding DefPathHashes can lead to correctness
+            // issues. We must abort compilation.
+            //
+            // The likelyhood of such a collision is very small, so actually
+            // running into one could be indicative of a poor hash function
+            // being used.
+            //
+            // See the documentation for DefPathHash for more information.
+            panic!(
+                "found DefPathHash collsion between {:?} and {:?}. \
+                    Compilation cannot continue.",
+                def_path1, def_path2
+            );
+        }
+
+        // Assert that all DefPathHashes correctly contain the local crate's
+        // StableCrateId
+        #[cfg(debug_assertions)]
+        if let Some(root) = self.def_path_hashes.get(CRATE_DEF_INDEX) {
+            assert!(def_path_hash.stable_crate_id() == root.stable_crate_id());
+        }
+
         index
     }
 
@@ -54,6 +87,7 @@ impl DefPathTable {
         hash
     }
 
+    /// Used by librustdoc for fake DefIds.
     pub fn num_def_ids(&self) -> usize {
         self.index_to_key.len()
     }
@@ -108,13 +142,10 @@ pub struct DefKey {
 }
 
 impl DefKey {
-    fn compute_stable_hash(&self, parent_hash: DefPathHash) -> DefPathHash {
+    pub(crate) fn compute_stable_hash(&self, parent: DefPathHash) -> DefPathHash {
         let mut hasher = StableHasher::new();
 
-        // We hash a `0u8` here to disambiguate between regular `DefPath` hashes,
-        // and the special "root_parent" below.
-        0u8.hash(&mut hasher);
-        parent_hash.hash(&mut hasher);
+        parent.hash(&mut hasher);
 
         let DisambiguatedDefPathData { ref data, disambiguator } = self.disambiguated_data;
 
@@ -127,19 +158,13 @@ impl DefKey {
 
         disambiguator.hash(&mut hasher);
 
-        DefPathHash(hasher.finish())
-    }
+        let local_hash: u64 = hasher.finish();
 
-    fn root_parent_stable_hash(
-        crate_name: &str,
-        crate_disambiguator: CrateDisambiguator,
-    ) -> DefPathHash {
-        let mut hasher = StableHasher::new();
-        // Disambiguate this from a regular `DefPath` hash; see `compute_stable_hash()` above.
-        1u8.hash(&mut hasher);
-        crate_name.hash(&mut hasher);
-        crate_disambiguator.hash(&mut hasher);
-        DefPathHash(hasher.finish())
+        // Construct the new DefPathHash, making sure that the `crate_id`
+        // portion of the hash is properly copied from the parent. This way the
+        // `crate_id` part will be recursively propagated from the root to all
+        // DefPathHashes in this DefPathTable.
+        DefPathHash::new(parent.stable_crate_id(), local_hash)
     }
 }
 
@@ -313,11 +338,6 @@ impl Definitions {
     }
 
     #[inline]
-    pub fn opt_local_def_id_to_hir_id(&self, id: LocalDefId) -> Option<hir::HirId> {
-        self.def_id_to_hir_id[id]
-    }
-
-    #[inline]
     pub fn opt_hir_id_to_local_def_id(&self, hir_id: hir::HirId) -> Option<LocalDefId> {
         self.hir_id_to_def_id.get(&hir_id).copied()
     }
@@ -332,7 +352,8 @@ impl Definitions {
             },
         };
 
-        let parent_hash = DefKey::root_parent_stable_hash(crate_name, crate_disambiguator);
+        let stable_crate_id = StableCrateId::new(crate_name, crate_disambiguator);
+        let parent_hash = DefPathHash::new(stable_crate_id, 0);
         let def_path_hash = key.compute_stable_hash(parent_hash);
 
         // Create the root definition.
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index 3cc501e423c..685429863fa 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -1,12 +1,12 @@
 // ignore-tidy-filelength
-use crate::def::{CtorKind, DefKind, Namespace, Res};
+use crate::def::{CtorKind, DefKind, Res};
 use crate::def_id::DefId;
 crate use crate::hir_id::HirId;
 use crate::{itemlikevisit, LangItem};
 
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_ast::{self as ast, CrateSugar, LlvmAsmDialect};
-use rustc_ast::{AttrVec, Attribute, FloatTy, IntTy, Label, LitKind, StrStyle, UintTy};
+use rustc_ast::{Attribute, FloatTy, IntTy, Label, LitKind, StrStyle, TraitObjectSyntax, UintTy};
 pub use rustc_ast::{BorrowKind, ImplPolarity, IsAuto};
 pub use rustc_ast::{CaptureBy, Movability, Mutability};
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
@@ -402,7 +402,7 @@ pub enum TraitBoundModifier {
 /// `typeck::collect::compute_bounds` matches these against
 /// the "special" built-in traits (see `middle::lang_items`) and
 /// detects `Copy`, `Send` and `Sync`.
-#[derive(Debug, HashStable_Generic)]
+#[derive(Clone, Debug, HashStable_Generic)]
 pub enum GenericBound<'hir> {
     Trait(PolyTraitRef<'hir>, TraitBoundModifier),
     // FIXME(davidtwco): Introduce `PolyTraitRef::LangItem`
@@ -469,7 +469,6 @@ pub enum GenericParamKind<'hir> {
 pub struct GenericParam<'hir> {
     pub hir_id: HirId,
     pub name: ParamName,
-    pub attrs: &'hir [Attribute],
     pub bounds: GenericBounds<'hir>,
     pub span: Span,
     pub pure_wrt_drop: bool,
@@ -626,14 +625,6 @@ pub struct ModuleItems {
     pub foreign_items: BTreeSet<ForeignItemId>,
 }
 
-/// A type representing only the top-level module.
-#[derive(Encodable, Debug, HashStable_Generic)]
-pub struct CrateItem<'hir> {
-    pub module: Mod<'hir>,
-    pub attrs: &'hir [Attribute],
-    pub span: Span,
-}
-
 /// The top-level data structure that stores the entire contents of
 /// the crate currently being compiled.
 ///
@@ -642,7 +633,7 @@ pub struct CrateItem<'hir> {
 /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/hir.html
 #[derive(Debug)]
 pub struct Crate<'hir> {
-    pub item: CrateItem<'hir>,
+    pub item: Mod<'hir>,
     pub exported_macros: &'hir [MacroDef<'hir>],
     // Attributes from non-exported macros, kept only for collecting the library feature list.
     pub non_exported_macro_attrs: &'hir [Attribute],
@@ -675,6 +666,9 @@ pub struct Crate<'hir> {
     pub proc_macros: Vec<HirId>,
 
     pub trait_map: BTreeMap<HirId, Vec<TraitCandidate>>,
+
+    /// Collected attributes from HIR nodes.
+    pub attrs: BTreeMap<HirId, &'hir [Attribute]>,
 }
 
 impl Crate<'hir> {
@@ -766,7 +760,6 @@ impl Crate<'_> {
 pub struct MacroDef<'hir> {
     pub ident: Ident,
     pub vis: Visibility<'hir>,
-    pub attrs: &'hir [Attribute],
     pub def_id: LocalDefId,
     pub span: Span,
     pub ast: ast::MacroDef,
@@ -882,7 +875,7 @@ impl<'hir> Pat<'hir> {
 /// are treated the same as` x: x, y: ref y, z: ref mut z`,
 /// except `is_shorthand` is true.
 #[derive(Debug, HashStable_Generic)]
-pub struct FieldPat<'hir> {
+pub struct PatField<'hir> {
     #[stable_hasher(ignore)]
     pub hir_id: HirId,
     /// The identifier for the field.
@@ -946,7 +939,7 @@ pub enum PatKind<'hir> {
 
     /// A struct or struct variant pattern (e.g., `Variant {x, y, ..}`).
     /// The `bool` is `true` in the presence of a `..`.
-    Struct(QPath<'hir>, &'hir [FieldPat<'hir>], bool),
+    Struct(QPath<'hir>, &'hir [PatField<'hir>], bool),
 
     /// A tuple struct/variant pattern `Variant(x, y, .., z)`.
     /// If the `..` pattern fragment is present, then `Option<usize>` denotes its position.
@@ -1166,16 +1159,6 @@ pub enum StmtKind<'hir> {
     Semi(&'hir Expr<'hir>),
 }
 
-impl<'hir> StmtKind<'hir> {
-    pub fn attrs(&self, get_item: impl FnOnce(ItemId) -> &'hir Item<'hir>) -> &'hir [Attribute] {
-        match *self {
-            StmtKind::Local(ref l) => &l.attrs,
-            StmtKind::Item(ref item_id) => &get_item(*item_id).attrs,
-            StmtKind::Expr(ref e) | StmtKind::Semi(ref e) => &e.attrs,
-        }
-    }
-}
-
 /// Represents a `let` statement (i.e., `let <pat>:<ty> = <expr>;`).
 #[derive(Debug, HashStable_Generic)]
 pub struct Local<'hir> {
@@ -1186,7 +1169,6 @@ pub struct Local<'hir> {
     pub init: Option<&'hir Expr<'hir>>,
     pub hir_id: HirId,
     pub span: Span,
-    pub attrs: AttrVec,
     /// Can be `ForLoopDesugar` if the `let` statement is part of a `for` loop
     /// desugaring. Otherwise will be `Normal`.
     pub source: LocalSource,
@@ -1199,7 +1181,6 @@ pub struct Arm<'hir> {
     #[stable_hasher(ignore)]
     pub hir_id: HirId,
     pub span: Span,
-    pub attrs: &'hir [Attribute],
     /// If this pattern and the optional guard matches, then `body` is evaluated.
     pub pat: &'hir Pat<'hir>,
     /// Optional guard clause.
@@ -1215,7 +1196,7 @@ pub enum Guard<'hir> {
 }
 
 #[derive(Debug, HashStable_Generic)]
-pub struct Field<'hir> {
+pub struct ExprField<'hir> {
     #[stable_hasher(ignore)]
     pub hir_id: HirId,
     pub ident: Ident,
@@ -1458,7 +1439,6 @@ pub struct AnonConst {
 pub struct Expr<'hir> {
     pub hir_id: HirId,
     pub kind: ExprKind<'hir>,
-    pub attrs: AttrVec,
     pub span: Span,
 }
 
@@ -1775,7 +1755,7 @@ pub enum ExprKind<'hir> {
     ///
     /// E.g., `Foo {x: 1, y: 2}`, or `Foo {x: 1, .. base}`,
     /// where `base` is the `Option<Expr>`.
-    Struct(&'hir QPath<'hir>, &'hir [Field<'hir>], Option<&'hir Expr<'hir>>),
+    Struct(&'hir QPath<'hir>, &'hir [ExprField<'hir>], Option<&'hir Expr<'hir>>),
 
     /// An array literal constructed from one repeated element.
     ///
@@ -1822,7 +1802,7 @@ impl<'hir> QPath<'hir> {
     pub fn span(&self) -> Span {
         match *self {
             QPath::Resolved(_, path) => path.span,
-            QPath::TypeRelative(_, ps) => ps.ident.span,
+            QPath::TypeRelative(qself, ps) => qself.span.to(ps.ident.span),
             QPath::LangItem(_, span) => span,
         }
     }
@@ -2040,7 +2020,6 @@ impl TraitItemId {
 pub struct TraitItem<'hir> {
     pub ident: Ident,
     pub def_id: LocalDefId,
-    pub attrs: &'hir [Attribute],
     pub generics: Generics<'hir>,
     pub kind: TraitItemKind<'hir>,
     pub span: Span,
@@ -2103,7 +2082,6 @@ pub struct ImplItem<'hir> {
     pub def_id: LocalDefId,
     pub vis: Visibility<'hir>,
     pub defaultness: Defaultness,
-    pub attrs: &'hir [Attribute],
     pub generics: Generics<'hir>,
     pub kind: ImplItemKind<'hir>,
     pub span: Span,
@@ -2133,15 +2111,6 @@ pub enum ImplItemKind<'hir> {
     TyAlias(&'hir Ty<'hir>),
 }
 
-impl ImplItemKind<'_> {
-    pub fn namespace(&self) -> Namespace {
-        match self {
-            ImplItemKind::TyAlias(..) => Namespace::TypeNS,
-            ImplItemKind::Const(..) | ImplItemKind::Fn(..) => Namespace::ValueNS,
-        }
-    }
-}
-
 // The name of the associated type for `Fn` return types.
 pub const FN_OUTPUT_NAME: Symbol = sym::Output;
 
@@ -2230,6 +2199,9 @@ impl PrimTy {
         Self::Str,
     ];
 
+    /// Like [`PrimTy::name`], but returns a &str instead of a symbol.
+    ///
+    /// Used by clippy.
     pub fn name_str(self) -> &'static str {
         match self {
             PrimTy::Int(i) => i.name_str(),
@@ -2306,7 +2278,9 @@ pub enum OpaqueTyOrigin {
     AsyncFn,
     /// `let _: impl Trait = ...`
     Binding,
-    /// Impl trait in type aliases, consts, statics, bounds.
+    /// type aliases: `type Foo = impl Trait;`
+    TyAlias,
+    /// Impl trait consts, statics, bounds.
     Misc,
 }
 
@@ -2340,7 +2314,7 @@ pub enum TyKind<'hir> {
     OpaqueDef(ItemId, &'hir [GenericArg<'hir>]),
     /// A trait object type `Bound1 + Bound2 + Bound3`
     /// where `Bound` is a trait or a lifetime.
-    TraitObject(&'hir [PolyTraitRef<'hir>], Lifetime),
+    TraitObject(&'hir [PolyTraitRef<'hir>], Lifetime, TraitObjectSyntax),
     /// Unused for now.
     Typeof(AnonConst),
     /// `TyKind::Infer` means the type should be inferred instead of it having been
@@ -2373,7 +2347,7 @@ pub enum InlineAsmOperand<'hir> {
         out_expr: Option<Expr<'hir>>,
     },
     Const {
-        expr: Expr<'hir>,
+        anon_const: AnonConst,
     },
     Sym {
         expr: Expr<'hir>,
@@ -2433,7 +2407,6 @@ pub struct LlvmInlineAsm<'hir> {
 /// Represents a parameter in a function header.
 #[derive(Debug, HashStable_Generic)]
 pub struct Param<'hir> {
-    pub attrs: &'hir [Attribute],
     pub hir_id: HirId,
     pub pat: &'hir Pat<'hir>,
     pub ty_span: Span,
@@ -2551,8 +2524,6 @@ pub struct Variant<'hir> {
     /// Name of the variant.
     #[stable_hasher(project(name))]
     pub ident: Ident,
-    /// Attributes of the variant.
-    pub attrs: &'hir [Attribute],
     /// Id of the variant (not the constructor, see `VariantData::ctor_hir_id()`).
     pub id: HirId,
     /// Fields and constructor id of the variant.
@@ -2585,7 +2556,7 @@ pub enum UseKind {
 /// that the `ref_id` is for. Note that `ref_id`'s value is not the `HirId` of the
 /// trait being referred to but just a unique `HirId` that serves as a key
 /// within the resolution map.
-#[derive(Debug, HashStable_Generic)]
+#[derive(Clone, Debug, HashStable_Generic)]
 pub struct TraitRef<'hir> {
     pub path: &'hir Path<'hir>,
     // Don't hash the `ref_id`. It is tracked via the thing it is used to access.
@@ -2604,7 +2575,7 @@ impl TraitRef<'_> {
     }
 }
 
-#[derive(Debug, HashStable_Generic)]
+#[derive(Clone, Debug, HashStable_Generic)]
 pub struct PolyTraitRef<'hir> {
     /// The `'a` in `for<'a> Foo<&'a T>`.
     pub bound_generic_params: &'hir [GenericParam<'hir>],
@@ -2639,17 +2610,16 @@ impl VisibilityKind<'_> {
 }
 
 #[derive(Debug, HashStable_Generic)]
-pub struct StructField<'hir> {
+pub struct FieldDef<'hir> {
     pub span: Span,
     #[stable_hasher(project(name))]
     pub ident: Ident,
     pub vis: Visibility<'hir>,
     pub hir_id: HirId,
     pub ty: &'hir Ty<'hir>,
-    pub attrs: &'hir [Attribute],
 }
 
-impl StructField<'_> {
+impl FieldDef<'_> {
     // Still necessary in couple of places
     pub fn is_positional(&self) -> bool {
         let first = self.ident.as_str().as_bytes()[0];
@@ -2663,11 +2633,11 @@ pub enum VariantData<'hir> {
     /// A struct variant.
     ///
     /// E.g., `Bar { .. }` as in `enum Foo { Bar { .. } }`.
-    Struct(&'hir [StructField<'hir>], /* recovered */ bool),
+    Struct(&'hir [FieldDef<'hir>], /* recovered */ bool),
     /// A tuple variant.
     ///
     /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`.
-    Tuple(&'hir [StructField<'hir>], HirId),
+    Tuple(&'hir [FieldDef<'hir>], HirId),
     /// A unit variant.
     ///
     /// E.g., `Bar = ..` as in `enum Foo { Bar = .. }`.
@@ -2676,7 +2646,7 @@ pub enum VariantData<'hir> {
 
 impl VariantData<'hir> {
     /// Return the fields of this variant.
-    pub fn fields(&self) -> &'hir [StructField<'hir>] {
+    pub fn fields(&self) -> &'hir [FieldDef<'hir>] {
         match *self {
             VariantData::Struct(ref fields, ..) | VariantData::Tuple(ref fields, ..) => fields,
             _ => &[],
@@ -2715,7 +2685,6 @@ impl ItemId {
 pub struct Item<'hir> {
     pub ident: Ident,
     pub def_id: LocalDefId,
-    pub attrs: &'hir [Attribute],
     pub kind: ItemKind<'hir>,
     pub vis: Visibility<'hir>,
     pub span: Span,
@@ -2932,7 +2901,6 @@ pub struct ForeignItemRef<'hir> {
 #[derive(Debug)]
 pub struct ForeignItem<'hir> {
     pub ident: Ident,
-    pub attrs: &'hir [Attribute],
     pub kind: ForeignItemKind<'hir>,
     pub def_id: LocalDefId,
     pub span: Span,
@@ -2986,7 +2954,7 @@ pub enum Node<'hir> {
     TraitItem(&'hir TraitItem<'hir>),
     ImplItem(&'hir ImplItem<'hir>),
     Variant(&'hir Variant<'hir>),
-    Field(&'hir StructField<'hir>),
+    Field(&'hir FieldDef<'hir>),
     AnonConst(&'hir AnonConst),
     Expr(&'hir Expr<'hir>),
     Stmt(&'hir Stmt<'hir>),
@@ -3008,7 +2976,7 @@ pub enum Node<'hir> {
     GenericParam(&'hir GenericParam<'hir>),
     Visibility(&'hir Visibility<'hir>),
 
-    Crate(&'hir CrateItem<'hir>),
+    Crate(&'hir Mod<'hir>),
 }
 
 impl<'hir> Node<'hir> {
@@ -3017,7 +2985,7 @@ impl<'hir> Node<'hir> {
             Node::TraitItem(TraitItem { ident, .. })
             | Node::ImplItem(ImplItem { ident, .. })
             | Node::ForeignItem(ForeignItem { ident, .. })
-            | Node::Field(StructField { ident, .. })
+            | Node::Field(FieldDef { ident, .. })
             | Node::Variant(Variant { ident, .. })
             | Node::MacroDef(MacroDef { ident, .. })
             | Node::Item(Item { ident, .. }) => Some(*ident),
@@ -3065,7 +3033,7 @@ impl<'hir> Node<'hir> {
             | Node::ImplItem(ImplItem { def_id, .. })
             | Node::ForeignItem(ForeignItem { def_id, .. })
             | Node::MacroDef(MacroDef { def_id, .. }) => Some(HirId::make_owner(*def_id)),
-            Node::Field(StructField { hir_id, .. })
+            Node::Field(FieldDef { hir_id, .. })
             | Node::AnonConst(AnonConst { hir_id, .. })
             | Node::Expr(Expr { hir_id, .. })
             | Node::Stmt(Stmt { hir_id, .. })
@@ -3088,16 +3056,16 @@ impl<'hir> Node<'hir> {
 }
 
 // Some nodes are used a lot. Make sure they don't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 mod size_asserts {
     rustc_data_structures::static_assert_size!(super::Block<'static>, 48);
-    rustc_data_structures::static_assert_size!(super::Expr<'static>, 72);
+    rustc_data_structures::static_assert_size!(super::Expr<'static>, 64);
     rustc_data_structures::static_assert_size!(super::Pat<'static>, 88);
     rustc_data_structures::static_assert_size!(super::QPath<'static>, 24);
     rustc_data_structures::static_assert_size!(super::Ty<'static>, 72);
 
-    rustc_data_structures::static_assert_size!(super::Item<'static>, 200);
-    rustc_data_structures::static_assert_size!(super::TraitItem<'static>, 144);
-    rustc_data_structures::static_assert_size!(super::ImplItem<'static>, 168);
-    rustc_data_structures::static_assert_size!(super::ForeignItem<'static>, 152);
+    rustc_data_structures::static_assert_size!(super::Item<'static>, 184);
+    rustc_data_structures::static_assert_size!(super::TraitItem<'static>, 128);
+    rustc_data_structures::static_assert_size!(super::ImplItem<'static>, 152);
+    rustc_data_structures::static_assert_size!(super::ForeignItem<'static>, 136);
 }
diff --git a/compiler/rustc_hir/src/hir_id.rs b/compiler/rustc_hir/src/hir_id.rs
index e24eb5e4490..0b25ebc27bd 100644
--- a/compiler/rustc_hir/src/hir_id.rs
+++ b/compiler/rustc_hir/src/hir_id.rs
@@ -1,4 +1,5 @@
 use crate::def_id::{LocalDefId, CRATE_DEF_INDEX};
+use rustc_index::vec::IndexVec;
 use std::fmt;
 
 /// Uniquely identifies a node in the HIR of the current crate. It is
@@ -61,3 +62,70 @@ pub const CRATE_HIR_ID: HirId = HirId {
     owner: LocalDefId { local_def_index: CRATE_DEF_INDEX },
     local_id: ItemLocalId::from_u32(0),
 };
+
+/// N.B. This collection is currently unused, but will be used by #72015 and future PRs.
+#[derive(Clone, Default, Debug, Encodable, Decodable)]
+pub struct HirIdVec<T> {
+    map: IndexVec<LocalDefId, IndexVec<ItemLocalId, T>>,
+}
+
+impl<T> HirIdVec<T> {
+    pub fn push_owner(&mut self, id: LocalDefId) {
+        self.map.ensure_contains_elem(id, IndexVec::new);
+    }
+
+    pub fn push(&mut self, id: HirId, value: T) {
+        if id.local_id == ItemLocalId::from_u32(0) {
+            self.push_owner(id.owner);
+        }
+        let submap = &mut self.map[id.owner];
+        let _ret_id = submap.push(value);
+        debug_assert_eq!(_ret_id, id.local_id);
+    }
+
+    pub fn push_sparse(&mut self, id: HirId, value: T)
+    where
+        T: Default,
+    {
+        self.map.ensure_contains_elem(id.owner, IndexVec::new);
+        let submap = &mut self.map[id.owner];
+        let i = id.local_id.index();
+        let len = submap.len();
+        if i >= len {
+            submap.extend(std::iter::repeat_with(T::default).take(i - len + 1));
+        }
+        submap[id.local_id] = value;
+    }
+
+    pub fn get(&self, id: HirId) -> Option<&T> {
+        self.map.get(id.owner)?.get(id.local_id)
+    }
+
+    pub fn get_owner(&self, id: LocalDefId) -> &IndexVec<ItemLocalId, T> {
+        &self.map[id]
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &T> {
+        self.map.iter().flat_map(|la| la.iter())
+    }
+
+    pub fn iter_enumerated(&self) -> impl Iterator<Item = (HirId, &T)> {
+        self.map.iter_enumerated().flat_map(|(owner, la)| {
+            la.iter_enumerated().map(move |(local_id, attr)| (HirId { owner, local_id }, attr))
+        })
+    }
+}
+
+impl<T> std::ops::Index<HirId> for HirIdVec<T> {
+    type Output = T;
+
+    fn index(&self, id: HirId) -> &T {
+        &self.map[id.owner][id.local_id]
+    }
+}
+
+impl<T> std::ops::IndexMut<HirId> for HirIdVec<T> {
+    fn index_mut(&mut self, id: HirId) -> &mut T {
+        &mut self.map[id.owner][id.local_id]
+    }
+}
diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs
index 6a2719c2d66..0ce04a77a50 100644
--- a/compiler/rustc_hir/src/intravisit.rs
+++ b/compiler/rustc_hir/src/intravisit.rs
@@ -98,32 +98,24 @@ where
     }
 }
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub enum FnKind<'a> {
     /// `#[xxx] pub async/const/extern "Abi" fn foo()`
-    ItemFn(Ident, &'a Generics<'a>, FnHeader, &'a Visibility<'a>, &'a [Attribute]),
+    ItemFn(Ident, &'a Generics<'a>, FnHeader, &'a Visibility<'a>),
 
     /// `fn foo(&self)`
-    Method(Ident, &'a FnSig<'a>, Option<&'a Visibility<'a>>, &'a [Attribute]),
+    Method(Ident, &'a FnSig<'a>, Option<&'a Visibility<'a>>),
 
     /// `|x, y| {}`
-    Closure(&'a [Attribute]),
+    Closure,
 }
 
 impl<'a> FnKind<'a> {
-    pub fn attrs(&self) -> &'a [Attribute] {
-        match *self {
-            FnKind::ItemFn(.., attrs) => attrs,
-            FnKind::Method(.., attrs) => attrs,
-            FnKind::Closure(attrs) => attrs,
-        }
-    }
-
     pub fn header(&self) -> Option<&FnHeader> {
         match *self {
-            FnKind::ItemFn(_, _, ref header, _, _) => Some(header),
-            FnKind::Method(_, ref sig, _, _) => Some(&sig.header),
-            FnKind::Closure(_) => None,
+            FnKind::ItemFn(_, _, ref header, _) => Some(header),
+            FnKind::Method(_, ref sig, _) => Some(&sig.header),
+            FnKind::Closure => None,
         }
     }
 }
@@ -374,6 +366,9 @@ pub trait Visitor<'v>: Sized {
     fn visit_generic_param(&mut self, p: &'v GenericParam<'v>) {
         walk_generic_param(self, p)
     }
+    fn visit_const_param_default(&mut self, _param: HirId, ct: &'v AnonConst) {
+        walk_const_param_default(self, ct)
+    }
     fn visit_generics(&mut self, g: &'v Generics<'v>) {
         walk_generics(self, g)
     }
@@ -423,8 +418,8 @@ pub trait Visitor<'v>: Sized {
     ) {
         walk_struct_def(self, s)
     }
-    fn visit_struct_field(&mut self, s: &'v StructField<'v>) {
-        walk_struct_field(self, s)
+    fn visit_field_def(&mut self, s: &'v FieldDef<'v>) {
+        walk_field_def(self, s)
     }
     fn visit_enum_def(
         &mut self,
@@ -466,7 +461,7 @@ pub trait Visitor<'v>: Sized {
     fn visit_assoc_type_binding(&mut self, type_binding: &'v TypeBinding<'v>) {
         walk_assoc_type_binding(self, type_binding)
     }
-    fn visit_attribute(&mut self, _attr: &'v Attribute) {}
+    fn visit_attribute(&mut self, _id: HirId, _attr: &'v Attribute) {}
     fn visit_macro_def(&mut self, macro_def: &'v MacroDef<'v>) {
         walk_macro_def(self, macro_def)
     }
@@ -483,15 +478,18 @@ pub trait Visitor<'v>: Sized {
 
 /// Walks the contents of a crate. See also `Crate::visit_all_items`.
 pub fn walk_crate<'v, V: Visitor<'v>>(visitor: &mut V, krate: &'v Crate<'v>) {
-    visitor.visit_mod(&krate.item.module, krate.item.span, CRATE_HIR_ID);
-    walk_list!(visitor, visit_attribute, krate.item.attrs);
+    visitor.visit_mod(&krate.item, krate.item.inner, CRATE_HIR_ID);
     walk_list!(visitor, visit_macro_def, krate.exported_macros);
+    for (&id, attrs) in krate.attrs.iter() {
+        for a in *attrs {
+            visitor.visit_attribute(id, a)
+        }
+    }
 }
 
 pub fn walk_macro_def<'v, V: Visitor<'v>>(visitor: &mut V, macro_def: &'v MacroDef<'v>) {
     visitor.visit_id(macro_def.hir_id());
     visitor.visit_ident(macro_def.ident);
-    walk_list!(visitor, visit_attribute, macro_def.attrs);
 }
 
 pub fn walk_mod<'v, V: Visitor<'v>>(visitor: &mut V, module: &'v Mod<'v>, mod_hir_id: HirId) {
@@ -510,7 +508,6 @@ pub fn walk_local<'v, V: Visitor<'v>>(visitor: &mut V, local: &'v Local<'v>) {
     // Intentionally visiting the expr first - the initialization expr
     // dominates the local's definition.
     walk_list!(visitor, visit_expr, &local.init);
-    walk_list!(visitor, visit_attribute, local.attrs.iter());
     visitor.visit_id(local.hir_id);
     visitor.visit_pat(&local.pat);
     walk_list!(visitor, visit_ty, &local.ty);
@@ -557,7 +554,6 @@ pub fn walk_trait_ref<'v, V: Visitor<'v>>(visitor: &mut V, trait_ref: &'v TraitR
 pub fn walk_param<'v, V: Visitor<'v>>(visitor: &mut V, param: &'v Param<'v>) {
     visitor.visit_id(param.hir_id);
     visitor.visit_pat(&param.pat);
-    walk_list!(visitor, visit_attribute, param.attrs);
 }
 
 pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) {
@@ -579,7 +575,7 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) {
             visitor.visit_nested_body(body);
         }
         ItemKind::Fn(ref sig, ref generics, body_id) => visitor.visit_fn(
-            FnKind::ItemFn(item.ident, generics, sig.header, &item.vis, &item.attrs),
+            FnKind::ItemFn(item.ident, generics, sig.header, &item.vis),
             &sig.decl,
             body_id,
             item.span,
@@ -652,7 +648,6 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) {
             walk_list!(visitor, visit_param_bound, bounds);
         }
     }
-    walk_list!(visitor, visit_attribute, item.attrs);
 }
 
 pub fn walk_use<'v, V: Visitor<'v>>(visitor: &mut V, path: &'v Path<'v>, hir_id: HirId) {
@@ -686,7 +681,6 @@ pub fn walk_variant<'v, V: Visitor<'v>>(
         variant.span,
     );
     walk_list!(visitor, visit_anon_const, &variant.disr_expr);
-    walk_list!(visitor, visit_attribute, variant.attrs);
 }
 
 pub fn walk_ty<'v, V: Visitor<'v>>(visitor: &mut V, typ: &'v Ty<'v>) {
@@ -718,7 +712,7 @@ pub fn walk_ty<'v, V: Visitor<'v>>(visitor: &mut V, typ: &'v Ty<'v>) {
             visitor.visit_ty(ty);
             visitor.visit_anon_const(length)
         }
-        TyKind::TraitObject(bounds, ref lifetime) => {
+        TyKind::TraitObject(bounds, ref lifetime, _syntax) => {
             for bound in bounds {
                 visitor.visit_poly_trait_ref(bound, TraitBoundModifier::None);
             }
@@ -851,8 +845,6 @@ pub fn walk_foreign_item<'v, V: Visitor<'v>>(visitor: &mut V, foreign_item: &'v
         ForeignItemKind::Static(ref typ, _) => visitor.visit_ty(typ),
         ForeignItemKind::Type => (),
     }
-
-    walk_list!(visitor, visit_attribute, foreign_item.attrs);
 }
 
 pub fn walk_param_bound<'v, V: Visitor<'v>>(visitor: &mut V, bound: &'v GenericBound<'v>) {
@@ -870,7 +862,6 @@ pub fn walk_param_bound<'v, V: Visitor<'v>>(visitor: &mut V, bound: &'v GenericB
 
 pub fn walk_generic_param<'v, V: Visitor<'v>>(visitor: &mut V, param: &'v GenericParam<'v>) {
     visitor.visit_id(param.hir_id);
-    walk_list!(visitor, visit_attribute, param.attrs);
     match param.name {
         ParamName::Plain(ident) => visitor.visit_ident(ident),
         ParamName::Error | ParamName::Fresh(_) => {}
@@ -881,13 +872,17 @@ pub fn walk_generic_param<'v, V: Visitor<'v>>(visitor: &mut V, param: &'v Generi
         GenericParamKind::Const { ref ty, ref default } => {
             visitor.visit_ty(ty);
             if let Some(ref default) = default {
-                visitor.visit_anon_const(default);
+                visitor.visit_const_param_default(param.hir_id, default);
             }
         }
     }
     walk_list!(visitor, visit_param_bound, param.bounds);
 }
 
+pub fn walk_const_param_default<'v, V: Visitor<'v>>(visitor: &mut V, ct: &'v AnonConst) {
+    visitor.visit_anon_const(ct)
+}
+
 pub fn walk_generics<'v, V: Visitor<'v>>(visitor: &mut V, generics: &'v Generics<'v>) {
     walk_list!(visitor, visit_generic_param, generics.params);
     walk_list!(visitor, visit_where_predicate, generics.where_clause.predicates);
@@ -940,7 +935,7 @@ pub fn walk_fn_kind<'v, V: Visitor<'v>>(visitor: &mut V, function_kind: FnKind<'
         FnKind::ItemFn(_, generics, ..) => {
             visitor.visit_generics(generics);
         }
-        FnKind::Method(..) | FnKind::Closure(_) => {}
+        FnKind::Method(..) | FnKind::Closure => {}
     }
 }
 
@@ -960,7 +955,6 @@ pub fn walk_fn<'v, V: Visitor<'v>>(
 
 pub fn walk_trait_item<'v, V: Visitor<'v>>(visitor: &mut V, trait_item: &'v TraitItem<'v>) {
     visitor.visit_ident(trait_item.ident);
-    walk_list!(visitor, visit_attribute, trait_item.attrs);
     visitor.visit_generics(&trait_item.generics);
     match trait_item.kind {
         TraitItemKind::Const(ref ty, default) => {
@@ -977,7 +971,7 @@ pub fn walk_trait_item<'v, V: Visitor<'v>>(visitor: &mut V, trait_item: &'v Trai
         }
         TraitItemKind::Fn(ref sig, TraitFn::Provided(body_id)) => {
             visitor.visit_fn(
-                FnKind::Method(trait_item.ident, sig, None, &trait_item.attrs),
+                FnKind::Method(trait_item.ident, sig, None),
                 &sig.decl,
                 body_id,
                 trait_item.span,
@@ -1003,21 +997,12 @@ pub fn walk_trait_item_ref<'v, V: Visitor<'v>>(visitor: &mut V, trait_item_ref:
 
 pub fn walk_impl_item<'v, V: Visitor<'v>>(visitor: &mut V, impl_item: &'v ImplItem<'v>) {
     // N.B., deliberately force a compilation error if/when new fields are added.
-    let ImplItem {
-        def_id: _,
-        ident,
-        ref vis,
-        ref defaultness,
-        attrs,
-        ref generics,
-        ref kind,
-        span: _,
-    } = *impl_item;
+    let ImplItem { def_id: _, ident, ref vis, ref defaultness, ref generics, ref kind, span: _ } =
+        *impl_item;
 
     visitor.visit_ident(ident);
     visitor.visit_vis(vis);
     visitor.visit_defaultness(defaultness);
-    walk_list!(visitor, visit_attribute, attrs);
     visitor.visit_generics(generics);
     match *kind {
         ImplItemKind::Const(ref ty, body) => {
@@ -1027,7 +1012,7 @@ pub fn walk_impl_item<'v, V: Visitor<'v>>(visitor: &mut V, impl_item: &'v ImplIt
         }
         ImplItemKind::Fn(ref sig, body_id) => {
             visitor.visit_fn(
-                FnKind::Method(impl_item.ident, sig, Some(&impl_item.vis), &impl_item.attrs),
+                FnKind::Method(impl_item.ident, sig, Some(&impl_item.vis)),
                 &sig.decl,
                 body_id,
                 impl_item.span,
@@ -1067,15 +1052,14 @@ pub fn walk_struct_def<'v, V: Visitor<'v>>(
     struct_definition: &'v VariantData<'v>,
 ) {
     walk_list!(visitor, visit_id, struct_definition.ctor_hir_id());
-    walk_list!(visitor, visit_struct_field, struct_definition.fields());
+    walk_list!(visitor, visit_field_def, struct_definition.fields());
 }
 
-pub fn walk_struct_field<'v, V: Visitor<'v>>(visitor: &mut V, struct_field: &'v StructField<'v>) {
-    visitor.visit_id(struct_field.hir_id);
-    visitor.visit_vis(&struct_field.vis);
-    visitor.visit_ident(struct_field.ident);
-    visitor.visit_ty(&struct_field.ty);
-    walk_list!(visitor, visit_attribute, struct_field.attrs);
+pub fn walk_field_def<'v, V: Visitor<'v>>(visitor: &mut V, field: &'v FieldDef<'v>) {
+    visitor.visit_id(field.hir_id);
+    visitor.visit_vis(&field.vis);
+    visitor.visit_ident(field.ident);
+    visitor.visit_ty(&field.ty);
 }
 
 pub fn walk_block<'v, V: Visitor<'v>>(visitor: &mut V, block: &'v Block<'v>) {
@@ -1102,7 +1086,6 @@ pub fn walk_anon_const<'v, V: Visitor<'v>>(visitor: &mut V, constant: &'v AnonCo
 
 pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>) {
     visitor.visit_id(expression.hir_id);
-    walk_list!(visitor, visit_attribute, expression.attrs.iter());
     match expression.kind {
         ExprKind::Box(ref subexpression) => visitor.visit_expr(subexpression),
         ExprKind::Array(subexpressions) => {
@@ -1162,7 +1145,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
         }
         ExprKind::Closure(_, ref function_declaration, body, _fn_decl_span, _gen) => visitor
             .visit_fn(
-                FnKind::Closure(&expression.attrs),
+                FnKind::Closure,
                 function_declaration,
                 body,
                 expression.span,
@@ -1206,7 +1189,6 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
                 match op {
                     InlineAsmOperand::In { expr, .. }
                     | InlineAsmOperand::InOut { expr, .. }
-                    | InlineAsmOperand::Const { expr, .. }
                     | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr),
                     InlineAsmOperand::Out { expr, .. } => {
                         if let Some(expr) = expr {
@@ -1219,6 +1201,9 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
                             visitor.visit_expr(out_expr);
                         }
                     }
+                    InlineAsmOperand::Const { anon_const, .. } => {
+                        visitor.visit_anon_const(anon_const)
+                    }
                 }
             }
         }
@@ -1246,7 +1231,6 @@ pub fn walk_arm<'v, V: Visitor<'v>>(visitor: &mut V, arm: &'v Arm<'v>) {
         }
     }
     visitor.visit_expr(&arm.body);
-    walk_list!(visitor, visit_attribute, arm.attrs);
 }
 
 pub fn walk_vis<'v, V: Visitor<'v>>(visitor: &mut V, vis: &'v Visibility<'v>) {
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 03524569ce7..498000db50f 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -38,27 +38,34 @@ macro_rules! expand_group {
 // So you probably just want to nip down to the end.
 macro_rules! language_item_table {
     (
-        $( $variant:ident $($group:expr)?, $name:expr, $method:ident, $target:expr; )*
+        $( $(#[$attr:meta])* $variant:ident $($group:expr)?, $module:ident :: $name:ident, $method:ident, $target:expr; )*
     ) => {
 
         enum_from_u32! {
             /// A representation of all the valid language items in Rust.
             #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable)]
             pub enum LangItem {
-                $($variant,)*
+                $(
+                    #[doc = concat!("The `", stringify!($name), "` lang item.")]
+                    ///
+                    $(#[$attr])*
+                    $variant,
+                )*
             }
         }
 
         impl LangItem {
             /// Returns the `name` symbol in `#[lang = "$name"]`.
-            /// For example, `LangItem::EqTraitLangItem`,
-            /// that is `#[lang = "eq"]` would result in `sym::eq`.
+            /// For example, [`LangItem::PartialEq`]`.name()`
+            /// would result in [`sym::eq`] since it is `#[lang = "eq"]`.
             pub fn name(self) -> Symbol {
                 match self {
-                    $( LangItem::$variant => $name, )*
+                    $( LangItem::$variant => $module::$name, )*
                 }
             }
 
+            /// The [group](LangItemGroup) that this lang item belongs to,
+            /// or `None` if it doesn't belong to a group.
             pub fn group(self) -> Option<LangItemGroup> {
                 use LangItemGroup::*;
                 match self {
@@ -67,15 +74,17 @@ macro_rules! language_item_table {
             }
         }
 
+        /// All of the language items, defined or not.
+        /// Defined lang items can come from the current crate or its dependencies.
         #[derive(HashStable_Generic, Debug)]
         pub struct LanguageItems {
-            /// Mappings from lang items to their possibly found `DefId`s.
-            /// The index corresponds to the order in `LangItem`.
+            /// Mappings from lang items to their possibly found [`DefId`]s.
+            /// The index corresponds to the order in [`LangItem`].
             pub items: Vec<Option<DefId>>,
             /// Lang items that were not found during collection.
             pub missing: Vec<LangItem>,
-            /// Mapping from `LangItemGroup` discriminants to all
-            /// `DefId`s of lang items in that group.
+            /// Mapping from [`LangItemGroup`] discriminants to all
+            /// [`DefId`]s of lang items in that group.
             pub groups: [Vec<DefId>; NUM_GROUPS],
         }
 
@@ -103,14 +112,13 @@ macro_rules! language_item_table {
                 self.items[it as usize].ok_or_else(|| format!("requires `{}` lang_item", it.name()))
             }
 
+            /// Returns the [`DefId`]s of all lang items in a group.
             pub fn group(&self, group: LangItemGroup) -> &[DefId] {
                 self.groups[group as usize].as_ref()
             }
 
             $(
-                /// Returns the corresponding `DefId` for the lang item if it
-                /// exists.
-                #[allow(dead_code)]
+                #[doc = concat!("Returns the [`DefId`] of the `", stringify!($name), "` lang item if it is defined.")]
                 pub fn $method(&self) -> Option<DefId> {
                     self.items[LangItem::$variant as usize]
                 }
@@ -120,7 +128,7 @@ macro_rules! language_item_table {
         /// A mapping from the name of the lang item to its order and the form it must be of.
         pub static ITEM_REFS: SyncLazy<FxHashMap<Symbol, (usize, Target)>> = SyncLazy::new(|| {
             let mut item_refs = FxHashMap::default();
-            $( item_refs.insert($name, (LangItem::$variant as usize, $target)); )*
+            $( item_refs.insert($module::$name, (LangItem::$variant as usize, $target)); )*
             item_refs
         });
 
@@ -140,7 +148,7 @@ impl<CTX> HashStable<CTX> for LangItem {
 ///
 /// About the `check_name` argument: passing in a `Session` would be simpler,
 /// because then we could call `Session::check_name` directly. But we want to
-/// avoid the need for `librustc_hir` to depend on `librustc_session`, so we
+/// avoid the need for `rustc_hir` to depend on `rustc_session`, so we
 /// use a closure instead.
 pub fn extract<'a, F>(check_name: F, attrs: &'a [ast::Attribute]) -> Option<(Symbol, Span)>
 where
@@ -190,15 +198,15 @@ language_item_table! {
 
     Sized,                   sym::sized,               sized_trait,                Target::Trait;
     Unsize,                  sym::unsize,              unsize_trait,               Target::Trait;
-    // Trait injected by #[derive(PartialEq)], (i.e. "Partial EQ").
+    /// Trait injected by `#[derive(PartialEq)]`, (i.e. "Partial EQ").
     StructuralPeq,           sym::structural_peq,      structural_peq_trait,       Target::Trait;
-    // Trait injected by #[derive(Eq)], (i.e. "Total EQ"; no, I will not apologize).
+    /// Trait injected by `#[derive(Eq)]`, (i.e. "Total EQ"; no, I will not apologize).
     StructuralTeq,           sym::structural_teq,      structural_teq_trait,       Target::Trait;
     Copy,                    sym::copy,                copy_trait,                 Target::Trait;
     Clone,                   sym::clone,               clone_trait,                Target::Trait;
     Sync,                    sym::sync,                sync_trait,                 Target::Trait;
     DiscriminantKind,        sym::discriminant_kind,   discriminant_kind_trait,    Target::Trait;
-    // The associated item of `trait DiscriminantKind`.
+    /// The associated item of the [`DiscriminantKind`] trait.
     Discriminant,            sym::discriminant_type,   discriminant_type,          Target::AssocTy;
 
     PointeeTrait,            sym::pointee_trait,       pointee_trait,              Target::Trait;
@@ -273,7 +281,7 @@ language_item_table! {
     PanicInfo,               sym::panic_info,          panic_info,                 Target::Struct;
     PanicLocation,           sym::panic_location,      panic_location,             Target::Struct;
     PanicImpl,               sym::panic_impl,          panic_impl,                 Target::Fn;
-    // libstd panic entry point. Necessary for const eval to be able to catch it
+    /// libstd panic entry point. Necessary for const eval to be able to catch it
     BeginPanic,              sym::begin_panic,         begin_panic_fn,             Target::Fn;
 
     ExchangeMalloc,          sym::exchange_malloc,     exchange_malloc_fn,         Target::Fn;
@@ -295,7 +303,7 @@ language_item_table! {
 
     MaybeUninit,             sym::maybe_uninit,        maybe_uninit,               Target::Union;
 
-    // Align offset for stride != 1; must not panic.
+    /// Align offset for stride != 1; must not panic.
     AlignOffset,             sym::align_offset,        align_offset_fn,            Target::Fn;
 
     Termination,             sym::termination,         termination,                Target::Trait;
diff --git a/compiler/rustc_hir/src/lib.rs b/compiler/rustc_hir/src/lib.rs
index c69a9b063ae..36a30900fb2 100644
--- a/compiler/rustc_hir/src/lib.rs
+++ b/compiler/rustc_hir/src/lib.rs
@@ -5,9 +5,10 @@
 #![feature(crate_visibility_modifier)]
 #![feature(const_fn)] // For the unsizing cast on `&[]`
 #![feature(const_panic)]
+#![feature(extended_key_value_attributes)]
 #![feature(in_band_lifetimes)]
 #![feature(once_cell)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 #[macro_use]
@@ -30,6 +31,9 @@ mod stable_hash_impls;
 mod target;
 pub mod weak_lang_items;
 
+#[cfg(test)]
+mod tests;
+
 pub use hir::*;
 pub use hir_id::*;
 pub use lang_items::{LangItem, LanguageItems};
diff --git a/compiler/rustc_hir/src/pat_util.rs b/compiler/rustc_hir/src/pat_util.rs
index 9e0a6aae242..b1f78a83e74 100644
--- a/compiler/rustc_hir/src/pat_util.rs
+++ b/compiler/rustc_hir/src/pat_util.rs
@@ -1,6 +1,7 @@
 use crate::def::{CtorOf, DefKind, Res};
 use crate::def_id::DefId;
 use crate::hir::{self, HirId, PatKind};
+use rustc_data_structures::stable_set::FxHashSet;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -89,26 +90,6 @@ impl hir::Pat<'_> {
         })
     }
 
-    /// Checks if the pattern contains any patterns that bind something to
-    /// an ident, e.g., `foo`, or `Foo(foo)` or `foo @ Bar(..)`.
-    pub fn contains_bindings(&self) -> bool {
-        self.satisfies(|p| matches!(p.kind, PatKind::Binding(..)))
-    }
-
-    /// Checks if the pattern satisfies the given predicate on some sub-pattern.
-    fn satisfies(&self, pred: impl Fn(&hir::Pat<'_>) -> bool) -> bool {
-        let mut satisfies = false;
-        self.walk_short(|p| {
-            if pred(p) {
-                satisfies = true;
-                false // Found one, can short circuit now.
-            } else {
-                true
-            }
-        });
-        satisfies
-    }
-
     pub fn simple_ident(&self) -> Option<Ident> {
         match self.kind {
             PatKind::Binding(
@@ -138,8 +119,10 @@ impl hir::Pat<'_> {
             }
             _ => true,
         });
-        variants.sort();
-        variants.dedup();
+        // We remove duplicates by inserting into a `FxHashSet` to avoid re-ordering
+        // the bounds
+        let mut duplicates = FxHashSet::default();
+        variants.retain(|def_id| duplicates.insert(*def_id));
         variants
     }
 
diff --git a/compiler/rustc_hir/src/stable_hash_impls.rs b/compiler/rustc_hir/src/stable_hash_impls.rs
index 5fb4b8a58c2..0232654aaa5 100644
--- a/compiler/rustc_hir/src/stable_hash_impls.rs
+++ b/compiler/rustc_hir/src/stable_hash_impls.rs
@@ -9,7 +9,7 @@ use rustc_span::def_id::{DefPathHash, LocalDefId};
 
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
-/// instead of implementing everything in librustc_middle.
+/// instead of implementing everything in `rustc_middle`.
 pub trait HashStableContext:
     rustc_ast::HashStableContext + rustc_target::HashStableContext
 {
@@ -139,11 +139,10 @@ impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for VisibilityKind<'_>
 
 impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for TraitItem<'_> {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let TraitItem { def_id: _, ident, ref attrs, ref generics, ref kind, span } = *self;
+        let TraitItem { def_id: _, ident, ref generics, ref kind, span } = *self;
 
         hcx.hash_hir_item_like(|hcx| {
             ident.name.hash_stable(hcx, hasher);
-            attrs.hash_stable(hcx, hasher);
             generics.hash_stable(hcx, hasher);
             kind.hash_stable(hcx, hasher);
             span.hash_stable(hcx, hasher);
@@ -153,22 +152,13 @@ impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for TraitItem<'_> {
 
 impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for ImplItem<'_> {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let ImplItem {
-            def_id: _,
-            ident,
-            ref vis,
-            defaultness,
-            ref attrs,
-            ref generics,
-            ref kind,
-            span,
-        } = *self;
+        let ImplItem { def_id: _, ident, ref vis, defaultness, ref generics, ref kind, span } =
+            *self;
 
         hcx.hash_hir_item_like(|hcx| {
             ident.name.hash_stable(hcx, hasher);
             vis.hash_stable(hcx, hasher);
             defaultness.hash_stable(hcx, hasher);
-            attrs.hash_stable(hcx, hasher);
             generics.hash_stable(hcx, hasher);
             kind.hash_stable(hcx, hasher);
             span.hash_stable(hcx, hasher);
@@ -178,11 +168,10 @@ impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for ImplItem<'_> {
 
 impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for ForeignItem<'_> {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let ForeignItem { def_id: _, ident, ref attrs, ref kind, span, ref vis } = *self;
+        let ForeignItem { def_id: _, ident, ref kind, span, ref vis } = *self;
 
         hcx.hash_hir_item_like(|hcx| {
             ident.name.hash_stable(hcx, hasher);
-            attrs.hash_stable(hcx, hasher);
             kind.hash_stable(hcx, hasher);
             span.hash_stable(hcx, hasher);
             vis.hash_stable(hcx, hasher);
@@ -192,11 +181,10 @@ impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for ForeignItem<'_> {
 
 impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for Item<'_> {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let Item { ident, ref attrs, def_id: _, ref kind, ref vis, span } = *self;
+        let Item { ident, def_id: _, ref kind, ref vis, span } = *self;
 
         hcx.hash_hir_item_like(|hcx| {
             ident.name.hash_stable(hcx, hasher);
-            attrs.hash_stable(hcx, hasher);
             kind.hash_stable(hcx, hasher);
             vis.hash_stable(hcx, hasher);
             span.hash_stable(hcx, hasher);
@@ -206,11 +194,10 @@ impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for Item<'_> {
 
 impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for MacroDef<'_> {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let MacroDef { ident, ref attrs, def_id: _, ref ast, ref vis, span } = *self;
+        let MacroDef { ident, def_id: _, ref ast, ref vis, span } = *self;
 
         hcx.hash_hir_item_like(|hcx| {
             ident.name.hash_stable(hcx, hasher);
-            attrs.hash_stable(hcx, hasher);
             ast.hash_stable(hcx, hasher);
             vis.hash_stable(hcx, hasher);
             span.hash_stable(hcx, hasher);
diff --git a/compiler/rustc_hir/src/tests.rs b/compiler/rustc_hir/src/tests.rs
new file mode 100644
index 00000000000..2aafc6afa23
--- /dev/null
+++ b/compiler/rustc_hir/src/tests.rs
@@ -0,0 +1,39 @@
+use crate::definitions::{DefKey, DefPathData, DisambiguatedDefPathData};
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_span::crate_disambiguator::CrateDisambiguator;
+use rustc_span::def_id::{DefPathHash, StableCrateId};
+
+#[test]
+fn def_path_hash_depends_on_crate_id() {
+    // This test makes sure that *both* halves of a DefPathHash depend on
+    // the crate-id of the defining crate. This is a desirable property
+    // because the crate-id can be more easily changed than the DefPath
+    // of an item, so, in the case of a crate-local DefPathHash collision,
+    // the user can simply "role the dice again" for all DefPathHashes in
+    // the crate by changing the crate disambiguator (e.g. via bumping the
+    // crate's version number).
+
+    let d0 = CrateDisambiguator::from(Fingerprint::new(12, 34));
+    let d1 = CrateDisambiguator::from(Fingerprint::new(56, 78));
+
+    let h0 = mk_test_hash("foo", d0);
+    let h1 = mk_test_hash("foo", d1);
+
+    assert_ne!(h0.stable_crate_id(), h1.stable_crate_id());
+    assert_ne!(h0.local_hash(), h1.local_hash());
+
+    fn mk_test_hash(crate_name: &str, crate_disambiguator: CrateDisambiguator) -> DefPathHash {
+        let stable_crate_id = StableCrateId::new(crate_name, crate_disambiguator);
+        let parent_hash = DefPathHash::new(stable_crate_id, 0);
+
+        let key = DefKey {
+            parent: None,
+            disambiguated_data: DisambiguatedDefPathData {
+                data: DefPathData::CrateRoot,
+                disambiguator: 0,
+            },
+        };
+
+        key.compute_stable_hash(parent_hash)
+    }
+}
diff --git a/compiler/rustc_hir/src/weak_lang_items.rs b/compiler/rustc_hir/src/weak_lang_items.rs
index b8cd15e7f00..58c3065240c 100644
--- a/compiler/rustc_hir/src/weak_lang_items.rs
+++ b/compiler/rustc_hir/src/weak_lang_items.rs
@@ -18,8 +18,8 @@ pub static WEAK_ITEMS_REFS: SyncLazy<StableMap<Symbol, LangItem>> = SyncLazy::ne
     map
 });
 
-/// The `check_name` argument avoids the need for `librustc_hir` to depend on
-/// `librustc_session`.
+/// The `check_name` argument avoids the need for `rustc_hir` to depend on
+/// `rustc_session`.
 pub fn link_name<'a, F>(check_name: F, attrs: &'a [ast::Attribute]) -> Option<Symbol>
 where
     F: Fn(&'a ast::Attribute, Symbol) -> bool
diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index 416918e3344..5820e7a2612 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -1,4 +1,4 @@
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 use rustc_ast as ast;
@@ -16,6 +16,7 @@ use rustc_target::spec::abi::Abi;
 
 use std::borrow::Cow;
 use std::cell::Cell;
+use std::collections::BTreeMap;
 use std::vec;
 
 pub fn id_to_string(map: &dyn rustc_hir::intravisit::Map<'_>, hir_id: hir::HirId) -> String {
@@ -82,6 +83,7 @@ impl PpAnn for &dyn rustc_hir::intravisit::Map<'_> {
 pub struct State<'a> {
     pub s: pp::Printer,
     comments: Option<Comments<'a>>,
+    attrs: &'a BTreeMap<hir::HirId, &'a [ast::Attribute]>,
     ann: &'a (dyn PpAnn + 'a),
 }
 
@@ -112,7 +114,7 @@ impl<'a> State<'a> {
             Node::Lifetime(a) => self.print_lifetime(&a),
             Node::Visibility(a) => self.print_visibility(&a),
             Node::GenericParam(_) => panic!("cannot print Node::GenericParam"),
-            Node::Field(_) => panic!("cannot print StructField"),
+            Node::Field(_) => panic!("cannot print Node::Field"),
             // These cases do not carry enough information in the
             // `hir_map` to reconstruct their full structure for pretty
             // printing.
@@ -163,12 +165,12 @@ pub fn print_crate<'a>(
     input: String,
     ann: &'a dyn PpAnn,
 ) -> String {
-    let mut s = State::new_from_input(sm, filename, input, ann);
+    let mut s = State::new_from_input(sm, filename, input, &krate.attrs, ann);
 
     // When printing the AST, we sometimes need to inject `#[no_std]` here.
     // Since you can't compile the HIR, it's not necessary.
 
-    s.print_mod(&krate.item.module, &krate.item.attrs);
+    s.print_mod(&krate.item, s.attrs(hir::CRATE_HIR_ID));
     s.print_remaining_comments();
     s.s.eof()
 }
@@ -178,9 +180,19 @@ impl<'a> State<'a> {
         sm: &'a SourceMap,
         filename: FileName,
         input: String,
+        attrs: &'a BTreeMap<hir::HirId, &[ast::Attribute]>,
         ann: &'a dyn PpAnn,
     ) -> State<'a> {
-        State { s: pp::mk_printer(), comments: Some(Comments::new(sm, filename, input)), ann }
+        State {
+            s: pp::mk_printer(),
+            comments: Some(Comments::new(sm, filename, input)),
+            attrs,
+            ann,
+        }
+    }
+
+    fn attrs(&self, id: hir::HirId) -> &'a [ast::Attribute] {
+        self.attrs.get(&id).map_or(&[], |la| *la)
     }
 }
 
@@ -188,7 +200,8 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
 where
     F: FnOnce(&mut State<'_>),
 {
-    let mut printer = State { s: pp::mk_printer(), comments: None, ann };
+    let mut printer =
+        State { s: pp::mk_printer(), comments: None, attrs: &BTreeMap::default(), ann };
     f(&mut printer);
     printer.s.eof()
 }
@@ -208,10 +221,6 @@ pub fn bounds_to_string<'b>(bounds: impl IntoIterator<Item = &'b hir::GenericBou
     to_string(NO_ANN, |s| s.print_bounds("", bounds))
 }
 
-pub fn param_to_string(arg: &hir::Param<'_>) -> String {
-    to_string(NO_ANN, |s| s.print_param(arg))
-}
-
 pub fn ty_to_string(ty: &hir::Ty<'_>) -> String {
     to_string(NO_ANN, |s| s.print_type(ty))
 }
@@ -397,7 +406,10 @@ impl<'a> State<'a> {
             }
             hir::TyKind::OpaqueDef(..) => self.s.word("/*impl Trait*/"),
             hir::TyKind::Path(ref qpath) => self.print_qpath(qpath, false),
-            hir::TyKind::TraitObject(bounds, ref lifetime) => {
+            hir::TyKind::TraitObject(bounds, ref lifetime, syntax) => {
+                if syntax == ast::TraitObjectSyntax::Dyn {
+                    self.word_space("dyn");
+                }
                 let mut first = true;
                 for bound in bounds {
                     if first {
@@ -441,7 +453,7 @@ impl<'a> State<'a> {
     pub fn print_foreign_item(&mut self, item: &hir::ForeignItem<'_>) {
         self.hardbreak_if_not_bol();
         self.maybe_print_comment(item.span.lo());
-        self.print_outer_attributes(&item.attrs);
+        self.print_outer_attributes(self.attrs(item.hir_id()));
         match item.kind {
             hir::ForeignItemKind::Fn(ref decl, ref arg_names, ref generics) => {
                 self.head("");
@@ -549,7 +561,8 @@ impl<'a> State<'a> {
     pub fn print_item(&mut self, item: &hir::Item<'_>) {
         self.hardbreak_if_not_bol();
         self.maybe_print_comment(item.span.lo());
-        self.print_outer_attributes(&item.attrs);
+        let attrs = self.attrs(item.hir_id());
+        self.print_outer_attributes(attrs);
         self.ann.pre(self, AnnNode::Item(item));
         match item.kind {
             hir::ItemKind::ExternCrate(orig_name) => {
@@ -634,14 +647,14 @@ impl<'a> State<'a> {
                 self.print_ident(item.ident);
                 self.nbsp();
                 self.bopen();
-                self.print_mod(_mod, &item.attrs);
+                self.print_mod(_mod, attrs);
                 self.bclose(item.span);
             }
             hir::ItemKind::ForeignMod { abi, items } => {
                 self.head("extern");
                 self.word_nbsp(abi.to_string());
                 self.bopen();
-                self.print_inner_attributes(item.attrs);
+                self.print_inner_attributes(self.attrs(item.hir_id()));
                 for item in items {
                     self.ann.nested(self, Nested::ForeignItem(item.id));
                 }
@@ -725,7 +738,7 @@ impl<'a> State<'a> {
 
                 self.s.space();
                 self.bopen();
-                self.print_inner_attributes(&item.attrs);
+                self.print_inner_attributes(attrs);
                 for impl_item in items {
                     self.ann.nested(self, Nested::ImplItem(impl_item.id));
                 }
@@ -822,7 +835,7 @@ impl<'a> State<'a> {
         for v in variants {
             self.space_if_not_bol();
             self.maybe_print_comment(v.span.lo());
-            self.print_outer_attributes(&v.attrs);
+            self.print_outer_attributes(self.attrs(v.id));
             self.ibox(INDENT_UNIT);
             self.print_variant(v);
             self.s.word(",");
@@ -876,7 +889,7 @@ impl<'a> State<'a> {
                     self.popen();
                     self.commasep(Inconsistent, struct_def.fields(), |s, field| {
                         s.maybe_print_comment(field.span.lo());
-                        s.print_outer_attributes(&field.attrs);
+                        s.print_outer_attributes(s.attrs(field.hir_id));
                         s.print_visibility(&field.vis);
                         s.print_type(&field.ty)
                     });
@@ -898,7 +911,7 @@ impl<'a> State<'a> {
                 for field in struct_def.fields() {
                     self.hardbreak_if_not_bol();
                     self.maybe_print_comment(field.span.lo());
-                    self.print_outer_attributes(&field.attrs);
+                    self.print_outer_attributes(self.attrs(field.hir_id));
                     self.print_visibility(&field.vis);
                     self.print_ident(field.ident);
                     self.word_nbsp(":");
@@ -937,7 +950,7 @@ impl<'a> State<'a> {
         self.ann.pre(self, AnnNode::SubItem(ti.hir_id()));
         self.hardbreak_if_not_bol();
         self.maybe_print_comment(ti.span.lo());
-        self.print_outer_attributes(&ti.attrs);
+        self.print_outer_attributes(self.attrs(ti.hir_id()));
         match ti.kind {
             hir::TraitItemKind::Const(ref ty, default) => {
                 let vis =
@@ -976,7 +989,7 @@ impl<'a> State<'a> {
         self.ann.pre(self, AnnNode::SubItem(ii.hir_id()));
         self.hardbreak_if_not_bol();
         self.maybe_print_comment(ii.span.lo());
-        self.print_outer_attributes(&ii.attrs);
+        self.print_outer_attributes(self.attrs(ii.hir_id()));
         self.print_defaultness(ii.defaultness);
 
         match ii.kind {
@@ -1193,7 +1206,7 @@ impl<'a> State<'a> {
     fn print_expr_struct(
         &mut self,
         qpath: &hir::QPath<'_>,
-        fields: &[hir::Field<'_>],
+        fields: &[hir::ExprField<'_>],
         wth: &Option<&hir::Expr<'_>>,
     ) {
         self.print_qpath(qpath, true);
@@ -1321,7 +1334,7 @@ impl<'a> State<'a> {
 
     pub fn print_expr(&mut self, expr: &hir::Expr<'_>) {
         self.maybe_print_comment(expr.span.lo());
-        self.print_outer_attributes(&expr.attrs);
+        self.print_outer_attributes(self.attrs(expr.hir_id));
         self.ibox(INDENT_UNIT);
         self.ann.pre(self, AnnNode::Expr(expr));
         match expr.kind {
@@ -1557,10 +1570,10 @@ impl<'a> State<'a> {
                                 None => s.word("_"),
                             }
                         }
-                        hir::InlineAsmOperand::Const { expr } => {
+                        hir::InlineAsmOperand::Const { anon_const } => {
                             s.word("const");
                             s.space();
-                            s.print_expr(expr);
+                            s.print_anon_const(anon_const);
                         }
                         hir::InlineAsmOperand::Sym { expr } => {
                             s.word("sym");
@@ -1684,21 +1697,10 @@ impl<'a> State<'a> {
         }
     }
 
-    pub fn print_usize(&mut self, i: usize) {
-        self.s.word(i.to_string())
-    }
-
     pub fn print_name(&mut self, name: Symbol) {
         self.print_ident(Ident::with_dummy_span(name))
     }
 
-    pub fn print_for_decl(&mut self, loc: &hir::Local<'_>, coll: &hir::Expr<'_>) {
-        self.print_local_decl(loc);
-        self.s.space();
-        self.word_space("in");
-        self.print_expr(coll)
-    }
-
     pub fn print_path(&mut self, path: &hir::Path<'_>, colons_before_params: bool) {
         self.maybe_print_comment(path.span.lo());
 
@@ -2020,20 +2022,20 @@ impl<'a> State<'a> {
     }
 
     pub fn print_param(&mut self, arg: &hir::Param<'_>) {
-        self.print_outer_attributes(&arg.attrs);
+        self.print_outer_attributes(self.attrs(arg.hir_id));
         self.print_pat(&arg.pat);
     }
 
     pub fn print_arm(&mut self, arm: &hir::Arm<'_>) {
         // I have no idea why this check is necessary, but here it
         // is :(
-        if arm.attrs.is_empty() {
+        if self.attrs(arm.hir_id).is_empty() {
             self.s.space();
         }
         self.cbox(INDENT_UNIT);
         self.ann.pre(self, AnnNode::Arm(arm));
         self.ibox(0);
-        self.print_outer_attributes(&arm.attrs);
+        self.print_outer_attributes(&self.attrs(arm.hir_id));
         self.print_pat(&arm.pat);
         self.s.space();
         if let Some(ref g) = arm.guard {
@@ -2249,8 +2251,10 @@ impl<'a> State<'a> {
             GenericParamKind::Const { ref ty, ref default } => {
                 self.word_space(":");
                 self.print_type(ty);
-                if let Some(ref _default) = default {
-                    // FIXME(const_generics_defaults): print the `default` value here
+                if let Some(ref default) = default {
+                    self.s.space();
+                    self.word_space("=");
+                    self.print_anon_const(&default)
                 }
             }
         }
@@ -2411,24 +2415,6 @@ impl<'a> State<'a> {
         }
     }
 
-    pub fn print_opt_abi_and_extern_if_nondefault(&mut self, opt_abi: Option<Abi>) {
-        match opt_abi {
-            Some(Abi::Rust) => {}
-            Some(abi) => {
-                self.word_nbsp("extern");
-                self.word_nbsp(abi.to_string())
-            }
-            None => {}
-        }
-    }
-
-    pub fn print_extern_opt_abi(&mut self, opt_abi: Option<Abi>) {
-        if let Some(abi) = opt_abi {
-            self.word_nbsp("extern");
-            self.word_nbsp(abi.to_string())
-        }
-    }
-
     pub fn print_fn_header_info(&mut self, header: hir::FnHeader, vis: &hir::Visibility<'_>) {
         self.s.word(visibility_qualified(vis, ""));
 
diff --git a/compiler/rustc_incremental/src/assert_dep_graph.rs b/compiler/rustc_incremental/src/assert_dep_graph.rs
index 1162379d3d9..b5680beae14 100644
--- a/compiler/rustc_incremental/src/assert_dep_graph.rs
+++ b/compiler/rustc_incremental/src/assert_dep_graph.rs
@@ -40,8 +40,9 @@ use rustc_graphviz as dot;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
-use rustc_middle::dep_graph::debug::{DepNodeFilter, EdgeFilter};
-use rustc_middle::dep_graph::{DepGraphQuery, DepKind, DepNode, DepNodeExt};
+use rustc_middle::dep_graph::{
+    DepGraphQuery, DepKind, DepNode, DepNodeExt, DepNodeFilter, EdgeFilter,
+};
 use rustc_middle::hir::map::Map;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::{sym, Symbol};
@@ -54,7 +55,11 @@ use std::io::{BufWriter, Write};
 pub fn assert_dep_graph(tcx: TyCtxt<'_>) {
     tcx.dep_graph.with_ignore(|| {
         if tcx.sess.opts.debugging_opts.dump_dep_graph {
-            dump_graph(tcx);
+            tcx.dep_graph.with_query(dump_graph);
+        }
+
+        if !tcx.sess.opts.debugging_opts.query_dep_graph {
+            return;
         }
 
         // if the `rustc_attrs` feature is not enabled, then the
@@ -68,7 +73,7 @@ pub fn assert_dep_graph(tcx: TyCtxt<'_>) {
         let (if_this_changed, then_this_would_need) = {
             let mut visitor =
                 IfThisChanged { tcx, if_this_changed: vec![], then_this_would_need: vec![] };
-            visitor.process_attrs(hir::CRATE_HIR_ID, &tcx.hir().krate().item.attrs);
+            visitor.process_attrs(hir::CRATE_HIR_ID);
             tcx.hir().krate().visit_all_item_likes(&mut visitor.as_deep_visitor());
             (visitor.if_this_changed, visitor.then_this_would_need)
         };
@@ -113,9 +118,10 @@ impl IfThisChanged<'tcx> {
         value
     }
 
-    fn process_attrs(&mut self, hir_id: hir::HirId, attrs: &[ast::Attribute]) {
+    fn process_attrs(&mut self, hir_id: hir::HirId) {
         let def_id = self.tcx.hir().local_def_id(hir_id);
         let def_path_hash = self.tcx.def_path_hash(def_id.to_def_id());
+        let attrs = self.tcx.hir().attrs(hir_id);
         for attr in attrs {
             if self.tcx.sess.check_name(attr, sym::rustc_if_this_changed) {
                 let dep_node_interned = self.argument(attr);
@@ -167,23 +173,23 @@ impl Visitor<'tcx> for IfThisChanged<'tcx> {
     }
 
     fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) {
-        self.process_attrs(item.hir_id(), &item.attrs);
+        self.process_attrs(item.hir_id());
         intravisit::walk_item(self, item);
     }
 
     fn visit_trait_item(&mut self, trait_item: &'tcx hir::TraitItem<'tcx>) {
-        self.process_attrs(trait_item.hir_id(), &trait_item.attrs);
+        self.process_attrs(trait_item.hir_id());
         intravisit::walk_trait_item(self, trait_item);
     }
 
     fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
-        self.process_attrs(impl_item.hir_id(), &impl_item.attrs);
+        self.process_attrs(impl_item.hir_id());
         intravisit::walk_impl_item(self, impl_item);
     }
 
-    fn visit_struct_field(&mut self, s: &'tcx hir::StructField<'tcx>) {
-        self.process_attrs(s.hir_id, &s.attrs);
-        intravisit::walk_struct_field(self, s);
+    fn visit_field_def(&mut self, s: &'tcx hir::FieldDef<'tcx>) {
+        self.process_attrs(s.hir_id);
+        intravisit::walk_field_def(self, s);
     }
 }
 
@@ -195,29 +201,29 @@ fn check_paths<'tcx>(tcx: TyCtxt<'tcx>, if_this_changed: &Sources, then_this_wou
         }
         return;
     }
-    let query = tcx.dep_graph.query();
-    for &(_, source_def_id, ref source_dep_node) in if_this_changed {
-        let dependents = query.transitive_predecessors(source_dep_node);
-        for &(target_span, ref target_pass, _, ref target_dep_node) in then_this_would_need {
-            if !dependents.contains(&target_dep_node) {
-                tcx.sess.span_err(
-                    target_span,
-                    &format!(
-                        "no path from `{}` to `{}`",
-                        tcx.def_path_str(source_def_id),
-                        target_pass
-                    ),
-                );
-            } else {
-                tcx.sess.span_err(target_span, "OK");
+    tcx.dep_graph.with_query(|query| {
+        for &(_, source_def_id, ref source_dep_node) in if_this_changed {
+            let dependents = query.transitive_predecessors(source_dep_node);
+            for &(target_span, ref target_pass, _, ref target_dep_node) in then_this_would_need {
+                if !dependents.contains(&target_dep_node) {
+                    tcx.sess.span_err(
+                        target_span,
+                        &format!(
+                            "no path from `{}` to `{}`",
+                            tcx.def_path_str(source_def_id),
+                            target_pass
+                        ),
+                    );
+                } else {
+                    tcx.sess.span_err(target_span, "OK");
+                }
             }
         }
-    }
+    });
 }
 
-fn dump_graph(tcx: TyCtxt<'_>) {
+fn dump_graph(query: &DepGraphQuery) {
     let path: String = env::var("RUST_DEP_GRAPH").unwrap_or_else(|_| "dep_graph".to_string());
-    let query = tcx.dep_graph.query();
 
     let nodes = match env::var("RUST_DEP_GRAPH_FILTER") {
         Ok(string) => {
diff --git a/compiler/rustc_incremental/src/assert_module_sources.rs b/compiler/rustc_incremental/src/assert_module_sources.rs
index 17d8ac9c882..5fb2c1cb9c9 100644
--- a/compiler/rustc_incremental/src/assert_module_sources.rs
+++ b/compiler/rustc_incremental/src/assert_module_sources.rs
@@ -44,7 +44,7 @@ pub fn assert_module_sources(tcx: TyCtxt<'_>) {
 
         let ams = AssertModuleSource { tcx, available_cgus };
 
-        for attr in tcx.hir().krate().item.attrs {
+        for attr in tcx.hir().attrs(rustc_hir::CRATE_HIR_ID) {
             ams.check_attr(attr);
         }
     })
diff --git a/compiler/rustc_incremental/src/lib.rs b/compiler/rustc_incremental/src/lib.rs
index 95456c07b10..f089cbcfca6 100644
--- a/compiler/rustc_incremental/src/lib.rs
+++ b/compiler/rustc_incremental/src/lib.rs
@@ -14,7 +14,7 @@ mod assert_dep_graph;
 pub mod assert_module_sources;
 mod persist;
 
-pub use assert_dep_graph::assert_dep_graph;
+use assert_dep_graph::assert_dep_graph;
 pub use persist::copy_cgu_workproduct_to_incr_comp_cache_dir;
 pub use persist::delete_workproduct_files;
 pub use persist::finalize_session_directory;
@@ -26,4 +26,4 @@ pub use persist::prepare_session_directory;
 pub use persist::save_dep_graph;
 pub use persist::save_work_product_index;
 pub use persist::LoadResult;
-pub use persist::{load_dep_graph, DepGraphFuture};
+pub use persist::{build_dep_graph, load_dep_graph, DepGraphFuture};
diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs
index 8a83149d732..e7bd488af8e 100644
--- a/compiler/rustc_incremental/src/persist/dirty_clean.rs
+++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs
@@ -14,7 +14,6 @@
 //! the required condition is not met.
 
 use rustc_ast::{self as ast, Attribute, NestedMetaItem};
-use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -74,16 +73,6 @@ const BASE_STRUCT: &[&str] =
     &[label_strs::generics_of, label_strs::predicates_of, label_strs::type_of];
 
 /// Trait definition `DepNode`s.
-const BASE_TRAIT_DEF: &[&str] = &[
-    label_strs::associated_item_def_ids,
-    label_strs::generics_of,
-    label_strs::object_safety_violations,
-    label_strs::predicates_of,
-    label_strs::specialization_graph_of,
-    label_strs::trait_def,
-    label_strs::trait_impls_of,
-];
-
 /// Extra `DepNode`s for functions and methods.
 const EXTRA_ASSOCIATED: &[&str] = &[label_strs::associated_item];
 
@@ -118,10 +107,6 @@ const LABELS_IMPL: &[&[&str]] = &[BASE_HIR, BASE_IMPL];
 /// Abstract data type (struct, enum, union) `DepNode`s.
 const LABELS_ADT: &[&[&str]] = &[BASE_HIR, BASE_STRUCT];
 
-/// Trait definition `DepNode`s.
-#[allow(dead_code)]
-const LABELS_TRAIT: &[&[&str]] = &[BASE_HIR, BASE_TRAIT_DEF];
-
 // FIXME: Struct/Enum/Unions Fields (there is currently no way to attach these)
 //
 // Fields are kind of separate from their containers, as they can change independently from
@@ -148,6 +133,10 @@ impl Assertion {
 }
 
 pub fn check_dirty_clean_annotations(tcx: TyCtxt<'_>) {
+    if !tcx.sess.opts.debugging_opts.query_dep_graph {
+        return;
+    }
+
     // can't add `#[rustc_dirty]` etc without opting in to this feature
     if !tcx.features().rustc_attrs {
         return;
@@ -168,7 +157,7 @@ pub fn check_dirty_clean_annotations(tcx: TyCtxt<'_>) {
         // Note that we cannot use the existing "unused attribute"-infrastructure
         // here, since that is running before codegen. This is also the reason why
         // all codegen-specific attributes are `AssumedUsed` in rustc_ast::feature_gate.
-        all_attrs.report_unchecked_attrs(&dirty_clean_visitor.checked_attrs);
+        all_attrs.report_unchecked_attrs(dirty_clean_visitor.checked_attrs);
     })
 }
 
@@ -391,10 +380,7 @@ impl DirtyCleanVisitor<'tcx> {
     fn assert_dirty(&self, item_span: Span, dep_node: DepNode) {
         debug!("assert_dirty({:?})", dep_node);
 
-        let current_fingerprint = self.get_fingerprint(&dep_node);
-        let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node);
-
-        if current_fingerprint == prev_fingerprint {
+        if self.tcx.dep_graph.is_green(&dep_node) {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx
                 .sess
@@ -402,28 +388,10 @@ impl DirtyCleanVisitor<'tcx> {
         }
     }
 
-    fn get_fingerprint(&self, dep_node: &DepNode) -> Option<Fingerprint> {
-        if self.tcx.dep_graph.dep_node_exists(dep_node) {
-            let dep_node_index = self.tcx.dep_graph.dep_node_index_of(dep_node);
-            Some(self.tcx.dep_graph.fingerprint_of(dep_node_index))
-        } else {
-            None
-        }
-    }
-
     fn assert_clean(&self, item_span: Span, dep_node: DepNode) {
         debug!("assert_clean({:?})", dep_node);
 
-        let current_fingerprint = self.get_fingerprint(&dep_node);
-        let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node);
-
-        // if the node wasn't previously evaluated and now is (or vice versa),
-        // then the node isn't actually clean or dirty.
-        if (current_fingerprint == None) ^ (prev_fingerprint == None) {
-            return;
-        }
-
-        if current_fingerprint != prev_fingerprint {
+        if self.tcx.dep_graph.is_red(&dep_node) {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx
                 .sess
@@ -535,13 +503,14 @@ impl FindAllAttrs<'_, 'tcx> {
         false
     }
 
-    fn report_unchecked_attrs(&self, checked_attrs: &FxHashSet<ast::AttrId>) {
+    fn report_unchecked_attrs(&self, mut checked_attrs: FxHashSet<ast::AttrId>) {
         for attr in &self.found_attrs {
             if !checked_attrs.contains(&attr.id) {
                 self.tcx.sess.span_err(
                     attr.span,
                     "found unchecked `#[rustc_dirty]` / `#[rustc_clean]` attribute",
                 );
+                checked_attrs.insert(attr.id);
             }
         }
     }
@@ -554,7 +523,7 @@ impl intravisit::Visitor<'tcx> for FindAllAttrs<'_, 'tcx> {
         intravisit::NestedVisitorMap::All(self.tcx.hir())
     }
 
-    fn visit_attribute(&mut self, attr: &'tcx Attribute) {
+    fn visit_attribute(&mut self, _: hir::HirId, attr: &'tcx Attribute) {
         if self.is_active_attr(attr) {
             self.found_attrs.push(attr);
         }
diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs
index 374a9eb41e5..b821ed6cff9 100644
--- a/compiler/rustc_incremental/src/persist/file_format.rs
+++ b/compiler/rustc_incremental/src/persist/file_format.rs
@@ -15,6 +15,7 @@ use std::io::{self, Read};
 use std::path::Path;
 
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
+use rustc_serialize::Encoder;
 
 /// The first few bytes of files generated by incremental compilation.
 const FILE_MAGIC: &[u8] = b"RSIC";
diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs
index c7a6c1195c5..30c6c408bc7 100644
--- a/compiler/rustc_incremental/src/persist/fs.rs
+++ b/compiler/rustc_incremental/src/persist/fs.rs
@@ -122,6 +122,7 @@ mod tests;
 
 const LOCK_FILE_EXT: &str = ".lock";
 const DEP_GRAPH_FILENAME: &str = "dep-graph.bin";
+const STAGING_DEP_GRAPH_FILENAME: &str = "dep-graph.part.bin";
 const WORK_PRODUCTS_FILENAME: &str = "work-products.bin";
 const QUERY_CACHE_FILENAME: &str = "query-cache.bin";
 
@@ -134,6 +135,9 @@ const INT_ENCODE_BASE: usize = base_n::CASE_INSENSITIVE;
 pub fn dep_graph_path(sess: &Session) -> PathBuf {
     in_incr_comp_dir_sess(sess, DEP_GRAPH_FILENAME)
 }
+pub fn staging_dep_graph_path(sess: &Session) -> PathBuf {
+    in_incr_comp_dir_sess(sess, STAGING_DEP_GRAPH_FILENAME)
+}
 pub fn dep_graph_path_from(incr_comp_session_dir: &Path) -> PathBuf {
     in_incr_comp_dir(incr_comp_session_dir, DEP_GRAPH_FILENAME)
 }
diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs
index 2b5649bb059..259e540c612 100644
--- a/compiler/rustc_incremental/src/persist/load.rs
+++ b/compiler/rustc_incremental/src/persist/load.rs
@@ -5,7 +5,7 @@ use rustc_hir::definitions::Definitions;
 use rustc_middle::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
 use rustc_middle::ty::query::OnDiskCache;
 use rustc_serialize::opaque::Decoder;
-use rustc_serialize::Decodable as RustcDecodable;
+use rustc_serialize::Decodable;
 use rustc_session::Session;
 use std::path::Path;
 
@@ -120,7 +120,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
             // Decode the list of work_products
             let mut work_product_decoder = Decoder::new(&work_products_data[..], start_pos);
             let work_products: Vec<SerializedWorkProduct> =
-                RustcDecodable::decode(&mut work_product_decoder).unwrap_or_else(|e| {
+                Decodable::decode(&mut work_product_decoder).unwrap_or_else(|e| {
                     let msg = format!(
                         "Error decoding `work-products` from incremental \
                                     compilation session directory: {}",
diff --git a/compiler/rustc_incremental/src/persist/mod.rs b/compiler/rustc_incremental/src/persist/mod.rs
index 8821b34b502..1336189bc0d 100644
--- a/compiler/rustc_incremental/src/persist/mod.rs
+++ b/compiler/rustc_incremental/src/persist/mod.rs
@@ -18,6 +18,7 @@ pub use fs::prepare_session_directory;
 pub use load::load_query_result_cache;
 pub use load::LoadResult;
 pub use load::{load_dep_graph, DepGraphFuture};
+pub use save::build_dep_graph;
 pub use save::save_dep_graph;
 pub use save::save_work_product_index;
 pub use work_product::copy_cgu_workproduct_to_incr_comp_cache_dir;
diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs
index 45d474b89b8..d558af3c1d5 100644
--- a/compiler/rustc_incremental/src/persist/save.rs
+++ b/compiler/rustc_incremental/src/persist/save.rs
@@ -1,6 +1,6 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::join;
-use rustc_middle::dep_graph::{DepGraph, WorkProduct, WorkProductId};
+use rustc_middle::dep_graph::{DepGraph, PreviousDepGraph, WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_serialize::Encodable as RustcEncodable;
@@ -15,6 +15,9 @@ use super::file_format;
 use super::fs::*;
 use super::work_product;
 
+/// Save and dump the DepGraph.
+///
+/// No query must be invoked after this function.
 pub fn save_dep_graph(tcx: TyCtxt<'_>) {
     debug!("save_dep_graph()");
     tcx.dep_graph.with_ignore(|| {
@@ -29,6 +32,14 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) {
 
         let query_cache_path = query_cache_path(sess);
         let dep_graph_path = dep_graph_path(sess);
+        let staging_dep_graph_path = staging_dep_graph_path(sess);
+
+        sess.time("assert_dep_graph", || crate::assert_dep_graph(tcx));
+        sess.time("check_dirty_clean", || dirty_clean::check_dirty_clean_annotations(tcx));
+
+        if sess.opts.debugging_opts.incremental_info {
+            tcx.dep_graph.print_incremental_info()
+        }
 
         join(
             move || {
@@ -36,16 +47,26 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) {
                     save_in(sess, query_cache_path, "query cache", |e| encode_query_cache(tcx, e));
                 });
             },
-            || {
+            move || {
                 sess.time("incr_comp_persist_dep_graph", || {
-                    save_in(sess, dep_graph_path, "dependency graph", |e| {
-                        sess.time("incr_comp_encode_dep_graph", || encode_dep_graph(tcx, e))
-                    });
+                    if let Err(err) = tcx.dep_graph.encode(&tcx.sess.prof) {
+                        sess.err(&format!(
+                            "failed to write dependency graph to `{}`: {}",
+                            staging_dep_graph_path.display(),
+                            err
+                        ));
+                    }
+                    if let Err(err) = fs::rename(&staging_dep_graph_path, &dep_graph_path) {
+                        sess.err(&format!(
+                            "failed to move dependency graph from `{}` to `{}`: {}",
+                            staging_dep_graph_path.display(),
+                            dep_graph_path.display(),
+                            err
+                        ));
+                    }
                 });
             },
         );
-
-        dirty_clean::check_dirty_clean_annotations(tcx);
     })
 }
 
@@ -92,7 +113,7 @@ pub fn save_work_product_index(
     });
 }
 
-fn save_in<F>(sess: &Session, path_buf: PathBuf, name: &str, encode: F)
+pub(crate) fn save_in<F>(sess: &Session, path_buf: PathBuf, name: &str, encode: F)
 where
     F: FnOnce(&mut FileEncoder) -> FileEncodeResult,
 {
@@ -144,21 +165,6 @@ where
     debug!("save: data written to disk successfully");
 }
 
-fn encode_dep_graph(tcx: TyCtxt<'_>, encoder: &mut FileEncoder) -> FileEncodeResult {
-    // First encode the commandline arguments hash
-    tcx.sess.opts.dep_tracking_hash().encode(encoder)?;
-
-    if tcx.sess.opts.debugging_opts.incremental_info {
-        tcx.dep_graph.print_incremental_info();
-    }
-
-    // There is a tiny window between printing the incremental info above and encoding the dep
-    // graph below in which the dep graph could change, thus making the printed incremental info
-    // slightly out of date. If this matters to you, please feel free to submit a patch. :)
-
-    tcx.sess.time("incr_comp_encode_serialized_dep_graph", || tcx.dep_graph.encode(encoder))
-}
-
 fn encode_work_product_index(
     work_products: &FxHashMap<WorkProductId, WorkProduct>,
     encoder: &mut FileEncoder,
@@ -177,3 +183,56 @@ fn encode_work_product_index(
 fn encode_query_cache(tcx: TyCtxt<'_>, encoder: &mut FileEncoder) -> FileEncodeResult {
     tcx.sess.time("incr_comp_serialize_result_cache", || tcx.serialize_query_result_cache(encoder))
 }
+
+pub fn build_dep_graph(
+    sess: &Session,
+    prev_graph: PreviousDepGraph,
+    prev_work_products: FxHashMap<WorkProductId, WorkProduct>,
+) -> Option<DepGraph> {
+    if sess.opts.incremental.is_none() {
+        // No incremental compilation.
+        return None;
+    }
+
+    // Stream the dep-graph to an alternate file, to avoid overwriting anything in case of errors.
+    let path_buf = staging_dep_graph_path(sess);
+
+    let mut encoder = match FileEncoder::new(&path_buf) {
+        Ok(encoder) => encoder,
+        Err(err) => {
+            sess.err(&format!(
+                "failed to create dependency graph at `{}`: {}",
+                path_buf.display(),
+                err
+            ));
+            return None;
+        }
+    };
+
+    if let Err(err) = file_format::write_file_header(&mut encoder, sess.is_nightly_build()) {
+        sess.err(&format!(
+            "failed to write dependency graph header to `{}`: {}",
+            path_buf.display(),
+            err
+        ));
+        return None;
+    }
+
+    // First encode the commandline arguments hash
+    if let Err(err) = sess.opts.dep_tracking_hash().encode(&mut encoder) {
+        sess.err(&format!(
+            "failed to write dependency graph hash `{}`: {}",
+            path_buf.display(),
+            err
+        ));
+        return None;
+    }
+
+    Some(DepGraph::new(
+        prev_graph,
+        prev_work_products,
+        encoder,
+        sess.opts.debugging_opts.query_dep_graph,
+        sess.opts.debugging_opts.incremental_info,
+    ))
+}
diff --git a/compiler/rustc_index/Cargo.toml b/compiler/rustc_index/Cargo.toml
index 6e1471df195..4b1f0b86475 100644
--- a/compiler/rustc_index/Cargo.toml
+++ b/compiler/rustc_index/Cargo.toml
@@ -8,6 +8,6 @@ edition = "2018"
 doctest = false
 
 [dependencies]
-arrayvec = { version = "0.5.1", default-features = false }
+arrayvec = { version = "0.7", default-features = false }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_macros = { path = "../rustc_macros" }
diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs
index 100824f4b94..d26ab1939e3 100644
--- a/compiler/rustc_index/src/bit_set.rs
+++ b/compiler/rustc_index/src/bit_set.rs
@@ -356,7 +356,7 @@ where
 {
     assert_eq!(out_vec.len(), in_vec.len());
     let mut changed = false;
-    for (out_elem, in_elem) in out_vec.iter_mut().zip(in_vec.iter()) {
+    for (out_elem, in_elem) in iter::zip(out_vec, in_vec) {
         let old_val = *out_elem;
         let new_val = op(old_val, *in_elem);
         *out_elem = new_val;
@@ -375,7 +375,7 @@ const SPARSE_MAX: usize = 8;
 #[derive(Clone, Debug)]
 pub struct SparseBitSet<T> {
     domain_size: usize,
-    elems: ArrayVec<[T; SPARSE_MAX]>,
+    elems: ArrayVec<T, SPARSE_MAX>,
 }
 
 impl<T: Idx> SparseBitSet<T> {
@@ -842,7 +842,7 @@ impl<R: Idx, C: Idx> BitMatrix<R, C> {
         let (write_start, write_end) = self.range(write);
         let words = &mut self.words[..];
         let mut changed = false;
-        for (read_index, write_index) in (read_start..read_end).zip(write_start..write_end) {
+        for (read_index, write_index) in iter::zip(read_start..read_end, write_start..write_end) {
             let word = words[write_index];
             let new_word = word | words[read_index];
             words[write_index] = new_word;
@@ -858,7 +858,7 @@ impl<R: Idx, C: Idx> BitMatrix<R, C> {
         assert_eq!(with.domain_size(), self.num_columns);
         let (write_start, write_end) = self.range(write);
         let mut changed = false;
-        for (read_index, write_index) in (0..with.words().len()).zip(write_start..write_end) {
+        for (read_index, write_index) in iter::zip(0..with.words().len(), write_start..write_end) {
             let word = self.words[write_index];
             let new_word = word | with.words()[read_index];
             self.words[write_index] = new_word;
diff --git a/compiler/rustc_index/src/lib.rs b/compiler/rustc_index/src/lib.rs
index 995034e81da..3ced3920cfd 100644
--- a/compiler/rustc_index/src/lib.rs
+++ b/compiler/rustc_index/src/lib.rs
@@ -2,6 +2,7 @@
 #![feature(const_fn)]
 #![feature(const_panic)]
 #![feature(extend_one)]
+#![feature(iter_zip)]
 #![feature(unboxed_closures)]
 #![feature(test)]
 #![feature(fn_traits)]
diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs
index 7cc4a5e9785..1b1a59a254e 100644
--- a/compiler/rustc_index/src/vec.rs
+++ b/compiler/rustc_index/src/vec.rs
@@ -124,7 +124,8 @@ macro_rules! newtype_index {
 
             #[inline]
             $v const fn from_usize(value: usize) -> Self {
-                assert!(value <= ($max as usize));
+                // FIXME: replace with `assert!(value <= ($max as usize));` once `const_panic` is stable
+                [()][(value > ($max as usize)) as usize];
                 unsafe {
                     Self::from_u32_unchecked(value as u32)
                 }
@@ -132,7 +133,8 @@ macro_rules! newtype_index {
 
             #[inline]
             $v const fn from_u32(value: u32) -> Self {
-                assert!(value <= $max);
+                // FIXME: replace with `assert!(value <= $max);` once `const_panic` is stable
+                [()][(value > $max) as usize];
                 unsafe {
                     Self::from_u32_unchecked(value)
                 }
@@ -695,9 +697,7 @@ impl<I: Idx, T> IndexVec<I, T> {
     pub fn convert_index_type<Ix: Idx>(self) -> IndexVec<Ix, T> {
         IndexVec { raw: self.raw, _marker: PhantomData }
     }
-}
 
-impl<I: Idx, T: Clone> IndexVec<I, T> {
     /// Grows the index vector so that it contains an entry for
     /// `elem`; if that is already true, then has no
     /// effect. Otherwise, inserts new values as needed by invoking
@@ -711,17 +711,19 @@ impl<I: Idx, T: Clone> IndexVec<I, T> {
     }
 
     #[inline]
-    pub fn resize(&mut self, new_len: usize, value: T) {
-        self.raw.resize(new_len, value)
-    }
-
-    #[inline]
     pub fn resize_to_elem(&mut self, elem: I, fill_value: impl FnMut() -> T) {
         let min_new_len = elem.index() + 1;
         self.raw.resize_with(min_new_len, fill_value);
     }
 }
 
+impl<I: Idx, T: Clone> IndexVec<I, T> {
+    #[inline]
+    pub fn resize(&mut self, new_len: usize, value: T) {
+        self.raw.resize(new_len, value)
+    }
+}
+
 impl<I: Idx, T: Ord> IndexVec<I, T> {
     #[inline]
     pub fn binary_search(&self, value: &T) -> Result<I, I> {
diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
index aa4fd055d5e..c68705da413 100644
--- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
+++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
@@ -293,7 +293,7 @@ impl<'cx, 'tcx> TypeFolder<'tcx> for Canonicalizer<'cx, 'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T>(&mut self, t: ty::Binder<T>) -> ty::Binder<T>
+    fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
     where
         T: TypeFoldable<'tcx>,
     {
@@ -621,7 +621,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
         let var = self.canonical_var(info, r.into());
-        let br = ty::BoundRegion { kind: ty::BrAnon(var.as_u32()) };
+        let br = ty::BoundRegion { var, kind: ty::BrAnon(var.as_u32()) };
         let region = ty::ReLateBound(self.binder_index, br);
         self.tcx().mk_region(region)
     }
diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs
index 2ec9b9e0be4..b8ecc949588 100644
--- a/compiler/rustc_infer/src/infer/canonical/query_response.rs
+++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs
@@ -27,6 +27,7 @@ use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind};
 use rustc_middle::ty::{self, BoundVar, Const, ToPredicate, Ty, TyCtxt};
 use std::fmt::Debug;
+use std::iter;
 
 impl<'cx, 'tcx> InferCtxt<'cx, 'tcx> {
     /// This method is meant to be invoked as the final step of a canonical query
@@ -418,7 +419,8 @@ impl<'cx, 'tcx> InferCtxt<'cx, 'tcx> {
 
         // In terms of our example above, we are iterating over pairs like:
         // [(?A, Vec<?0>), ('static, '?1), (?B, ?0)]
-        for (original_value, result_value) in original_values.var_values.iter().zip(result_values) {
+        for (original_value, result_value) in iter::zip(&original_values.var_values, result_values)
+        {
             match result_value.unpack() {
                 GenericArgKind::Type(result_value) => {
                     // e.g., here `result_value` might be `?0` in the example above...
@@ -437,7 +439,7 @@ impl<'cx, 'tcx> InferCtxt<'cx, 'tcx> {
 
                         // We only allow a `ty::INNERMOST` index in substitutions.
                         assert_eq!(debruijn, ty::INNERMOST);
-                        opt_values[br.assert_bound_var()] = Some(*original_value);
+                        opt_values[br.var] = Some(*original_value);
                     }
                 }
                 GenericArgKind::Const(result_value) => {
diff --git a/compiler/rustc_infer/src/infer/canonical/substitute.rs b/compiler/rustc_infer/src/infer/canonical/substitute.rs
index 387f480814a..553a11d4393 100644
--- a/compiler/rustc_infer/src/infer/canonical/substitute.rs
+++ b/compiler/rustc_infer/src/infer/canonical/substitute.rs
@@ -71,11 +71,10 @@ where
     if var_values.var_values.is_empty() {
         value
     } else {
-        let fld_r =
-            |br: ty::BoundRegion| match var_values.var_values[br.assert_bound_var()].unpack() {
-                GenericArgKind::Lifetime(l) => l,
-                r => bug!("{:?} is a region but value is {:?}", br, r),
-            };
+        let fld_r = |br: ty::BoundRegion| match var_values.var_values[br.var].unpack() {
+            GenericArgKind::Lifetime(l) => l,
+            r => bug!("{:?} is a region but value is {:?}", br, r),
+        };
 
         let fld_t = |bound_ty: ty::BoundTy| match var_values.var_values[bound_ty.var].unpack() {
             GenericArgKind::Type(ty) => ty,
diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs
index 5e11932eafc..30214e94203 100644
--- a/compiler/rustc_infer/src/infer/combine.rs
+++ b/compiler/rustc_infer/src/infer/combine.rs
@@ -191,7 +191,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
     ///
     /// This also tests if the given const `ct` contains an inference variable which was previously
     /// unioned with `target_vid`. If this is the case, inferring `target_vid` to `ct`
-    /// would result in an infinite type as we continously replace an inference variable
+    /// would result in an infinite type as we continuously replace an inference variable
     /// in `ct` with `ct` itself.
     ///
     /// This is especially important as unevaluated consts use their parents generics.
@@ -543,15 +543,11 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
         true
     }
 
-    fn visit_ct_substs(&self) -> bool {
-        true
-    }
-
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
@@ -737,6 +733,16 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
                     }
                 }
             }
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+                if self.tcx().lazy_normalization() =>
+            {
+                assert_eq!(promoted, None);
+                let substs = self.relate_with_variance(ty::Variance::Invariant, substs, substs)?;
+                Ok(self.tcx().mk_const(ty::Const {
+                    ty: c.ty,
+                    val: ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }),
+                }))
+            }
             _ => relate::super_relate_consts(self, c, c),
         }
     }
@@ -822,10 +828,6 @@ impl TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
         true
     }
 
-    fn visit_ct_substs(&self) -> bool {
-        true
-    }
-
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         _variance: ty::Variance,
@@ -838,9 +840,9 @@ impl TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
@@ -959,6 +961,16 @@ impl TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
                     }
                 }
             }
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+                if self.tcx().lazy_normalization() =>
+            {
+                assert_eq!(promoted, None);
+                let substs = self.relate_with_variance(ty::Variance::Invariant, substs, substs)?;
+                Ok(self.tcx().mk_const(ty::Const {
+                    ty: c.ty,
+                    val: ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }),
+                }))
+            }
             _ => relate::super_relate_consts(self, c, c),
         }
     }
diff --git a/compiler/rustc_infer/src/infer/equate.rs b/compiler/rustc_infer/src/infer/equate.rs
index 7c388b5503e..45ba50bb634 100644
--- a/compiler/rustc_infer/src/infer/equate.rs
+++ b/compiler/rustc_infer/src/infer/equate.rs
@@ -124,9 +124,9 @@ impl TypeRelation<'tcx> for Equate<'combine, 'infcx, 'tcx> {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
index eeff48a6395..a91bd9ce2ff 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
@@ -67,13 +67,13 @@ use rustc_hir::{Item, ItemKind, Node};
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::{
     self,
-    subst::{Subst, SubstsRef},
+    subst::{GenericArgKind, Subst, SubstsRef},
     Region, Ty, TyCtxt, TypeFoldable,
 };
 use rustc_span::{sym, BytePos, DesugaringKind, Pos, Span};
 use rustc_target::spec::abi;
 use std::ops::ControlFlow;
-use std::{cmp, fmt};
+use std::{cmp, fmt, iter};
 
 mod note;
 
@@ -514,7 +514,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
 
             fn print_dyn_existential(
                 self,
-                _predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+                _predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
             ) -> Result<Self::DynExistential, Self::Error> {
                 Err(NonTrivialPath)
             }
@@ -957,33 +957,27 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
     ) -> SubstsRef<'tcx> {
         let generics = self.tcx.generics_of(def_id);
         let mut num_supplied_defaults = 0;
-        let mut type_params = generics
-            .params
-            .iter()
-            .rev()
-            .filter_map(|param| match param.kind {
-                ty::GenericParamDefKind::Lifetime => None,
-                ty::GenericParamDefKind::Type { has_default, .. } => {
-                    Some((param.def_id, has_default))
-                }
-                ty::GenericParamDefKind::Const => None, // FIXME(const_generics_defaults)
-            })
-            .peekable();
-        let has_default = {
-            let has_default = type_params.peek().map(|(_, has_default)| has_default);
-            *has_default.unwrap_or(&false)
-        };
-        if has_default {
-            let types = substs.types().rev();
-            for ((def_id, has_default), actual) in type_params.zip(types) {
-                if !has_default {
-                    break;
+
+        let default_params = generics.params.iter().rev().filter_map(|param| match param.kind {
+            ty::GenericParamDefKind::Type { has_default: true, .. } => Some(param.def_id),
+            ty::GenericParamDefKind::Const { has_default: true } => Some(param.def_id),
+            _ => None,
+        });
+        for (def_id, actual) in iter::zip(default_params, substs.iter().rev()) {
+            match actual.unpack() {
+                GenericArgKind::Const(c) => {
+                    if self.tcx.const_param_default(def_id).subst(self.tcx, substs) != c {
+                        break;
+                    }
                 }
-                if self.tcx.type_of(def_id).subst(self.tcx, substs) != actual {
-                    break;
+                GenericArgKind::Type(ty) => {
+                    if self.tcx.type_of(def_id).subst(self.tcx, substs) != ty {
+                        break;
+                    }
                 }
-                num_supplied_defaults += 1;
+                _ => break,
             }
+            num_supplied_defaults += 1;
         }
         let len = generics.params.len();
         let mut generics = generics.clone();
@@ -1046,7 +1040,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
         let len1 = sig1.inputs().len();
         let len2 = sig2.inputs().len();
         if len1 == len2 {
-            for (i, (l, r)) in sig1.inputs().iter().zip(sig2.inputs().iter()).enumerate() {
+            for (i, (l, r)) in iter::zip(sig1.inputs(), sig2.inputs()).enumerate() {
                 let (x1, x2) = self.cmp(l, r);
                 (values.0).0.extend(x1.0);
                 (values.1).0.extend(x2.0);
@@ -1167,12 +1161,10 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
                     let common_len = cmp::min(len1, len2);
                     let remainder1: Vec<_> = sub1.types().skip(common_len).collect();
                     let remainder2: Vec<_> = sub2.types().skip(common_len).collect();
-                    let common_default_params = remainder1
-                        .iter()
-                        .rev()
-                        .zip(remainder2.iter().rev())
-                        .filter(|(a, b)| a == b)
-                        .count();
+                    let common_default_params =
+                        iter::zip(remainder1.iter().rev(), remainder2.iter().rev())
+                            .filter(|(a, b)| a == b)
+                            .count();
                     let len = sub1.len() - common_default_params;
                     let consts_offset = len - sub1.consts().count();
 
@@ -1303,12 +1295,11 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
 
                     const SEPARATOR: &str = "::";
                     let separator_len = SEPARATOR.len();
-                    let split_idx: usize = t1_str
-                        .split(SEPARATOR)
-                        .zip(t2_str.split(SEPARATOR))
-                        .take_while(|(mod1_str, mod2_str)| mod1_str == mod2_str)
-                        .map(|(mod_str, _)| mod_str.len() + separator_len)
-                        .sum();
+                    let split_idx: usize =
+                        iter::zip(t1_str.split(SEPARATOR), t2_str.split(SEPARATOR))
+                            .take_while(|(mod1_str, mod2_str)| mod1_str == mod2_str)
+                            .map(|(mod_str, _)| mod_str.len() + separator_len)
+                            .sum();
 
                     debug!(
                         "cmp: separator_len={}, split_idx={}, min_len={}",
@@ -1913,7 +1904,9 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
                         .find_map(|(path, msg)| (&path_str == path).then_some(msg))
                     {
                         let mut show_suggestion = true;
-                        for (exp_ty, found_ty) in exp_substs.types().zip(found_substs.types()) {
+                        for (exp_ty, found_ty) in
+                            iter::zip(exp_substs.types(), found_substs.types())
+                        {
                             match *exp_ty.kind() {
                                 ty::Ref(_, exp_ty, _) => {
                                     match (exp_ty.kind(), found_ty.kind()) {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
index d533e267fd7..d9a1193aac4 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
@@ -287,6 +287,7 @@ pub struct InferenceDiagnosticsData {
 pub struct InferenceDiagnosticsParentData {
     pub prefix: &'static str,
     pub name: String,
+    pub def_id: DefId,
 }
 
 pub enum UnderspecifiedArgKind {
@@ -328,6 +329,7 @@ impl InferenceDiagnosticsParentData {
         Some(InferenceDiagnosticsParentData {
             prefix: tcx.def_kind(parent_def_id).descr(parent_def_id),
             name: parent_name,
+            def_id: parent_def_id,
         })
     }
 }
@@ -754,12 +756,30 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
             if let (UnderspecifiedArgKind::Const { .. }, Some(parent_data)) =
                 (&arg_data.kind, &arg_data.parent)
             {
-                err.span_suggestion_verbose(
-                    span,
-                    "consider specifying the const argument",
-                    format!("{}::<{}>", parent_data.name, arg_data.name),
-                    Applicability::MaybeIncorrect,
-                );
+                let has_impl_trait =
+                    self.tcx.generics_of(parent_data.def_id).params.iter().any(|param| {
+                        matches!(
+                            param.kind,
+                            ty::GenericParamDefKind::Type {
+                                synthetic: Some(
+                                    hir::SyntheticTyParamKind::ImplTrait
+                                        | hir::SyntheticTyParamKind::FromAttr,
+                                ),
+                                ..
+                            }
+                        )
+                    });
+
+                // (#83606): Do not emit a suggestion if the parent has an `impl Trait`
+                // as an argument otherwise it will cause the E0282 error.
+                if !has_impl_trait {
+                    err.span_suggestion_verbose(
+                        span,
+                        "consider specifying the const argument",
+                        format!("{}::<{}>", parent_data.name, arg_data.name),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
             }
 
             err.span_label(
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
index d9ab8319045..58eb1e9aa12 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
@@ -99,7 +99,7 @@ impl Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                 return;
             }
 
-            hir::TyKind::TraitObject(bounds, _) => {
+            hir::TyKind::TraitObject(bounds, ..) => {
                 for bound in bounds {
                     self.current_index.shift_in(1);
                     self.visit_poly_trait_ref(bound, hir::TraitBoundModifier::None);
@@ -115,7 +115,7 @@ impl Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                     // error. We will then search the function parameters for a bound
                     // region at the right depth with the same index
                     (
-                        Some(rl::Region::LateBoundAnon(debruijn_index, anon_index)),
+                        Some(rl::Region::LateBoundAnon(debruijn_index, _, anon_index)),
                         ty::BrAnon(br_index),
                     ) => {
                         debug!(
@@ -143,7 +143,7 @@ impl Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                     // error. We will then search the function parameters for a bound
                     // region at the right depth with the same index
                     (
-                        Some(rl::Region::LateBound(debruijn_index, id, _)),
+                        Some(rl::Region::LateBound(debruijn_index, _, id, _)),
                         ty::BrNamed(def_id, _),
                     ) => {
                         debug!(
@@ -162,8 +162,8 @@ impl Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                             rl::Region::Static
                             | rl::Region::Free(_, _)
                             | rl::Region::EarlyBound(_, _, _)
-                            | rl::Region::LateBound(_, _, _)
-                            | rl::Region::LateBoundAnon(_, _),
+                            | rl::Region::LateBound(_, _, _, _)
+                            | rl::Region::LateBoundAnon(_, _, _),
                         )
                         | None,
                         _,
@@ -217,7 +217,10 @@ impl Visitor<'tcx> for TyPathVisitor<'tcx> {
     fn visit_lifetime(&mut self, lifetime: &hir::Lifetime) {
         match (self.tcx.named_region(lifetime.hir_id), self.bound_region) {
             // the lifetime of the TyPath!
-            (Some(rl::Region::LateBoundAnon(debruijn_index, anon_index)), ty::BrAnon(br_index)) => {
+            (
+                Some(rl::Region::LateBoundAnon(debruijn_index, _, anon_index)),
+                ty::BrAnon(br_index),
+            ) => {
                 if debruijn_index == self.current_index && anon_index == br_index {
                     self.found_it = true;
                     return;
@@ -232,7 +235,7 @@ impl Visitor<'tcx> for TyPathVisitor<'tcx> {
                 }
             }
 
-            (Some(rl::Region::LateBound(debruijn_index, id, _)), ty::BrNamed(def_id, _)) => {
+            (Some(rl::Region::LateBound(debruijn_index, _, id, _)), ty::BrNamed(def_id, _)) => {
                 debug!("FindNestedTypeVisitor::visit_ty: LateBound depth = {:?}", debruijn_index,);
                 debug!("id={:?}", id);
                 debug!("def_id={:?}", def_id);
@@ -246,8 +249,8 @@ impl Visitor<'tcx> for TyPathVisitor<'tcx> {
                 Some(
                     rl::Region::Static
                     | rl::Region::EarlyBound(_, _, _)
-                    | rl::Region::LateBound(_, _, _)
-                    | rl::Region::LateBoundAnon(_, _)
+                    | rl::Region::LateBound(_, _, _, _)
+                    | rl::Region::LateBoundAnon(_, _, _)
                     | rl::Region::Free(_, _),
                 )
                 | None,
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
index fa0d5b83013..1e926989263 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
@@ -292,7 +292,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
                         );
                     }
                 }
-                TyKind::TraitObject(_, lt) => match lt.name {
+                TyKind::TraitObject(_, lt, _) => match lt.name {
                     LifetimeName::ImplicitObjectLifetimeDefault => {
                         err.span_suggestion_verbose(
                             fn_return.span.shrink_to_hi(),
@@ -498,6 +498,7 @@ impl<'tcx> Visitor<'tcx> for HirTraitObjectVisitor {
         if let TyKind::TraitObject(
             poly_trait_refs,
             Lifetime { name: LifetimeName::ImplicitObjectLifetimeDefault, .. },
+            _,
         ) = t.kind
         {
             for ptr in poly_trait_refs {
diff --git a/compiler/rustc_infer/src/infer/glb.rs b/compiler/rustc_infer/src/infer/glb.rs
index ccba904df9e..02662043dba 100644
--- a/compiler/rustc_infer/src/infer/glb.rs
+++ b/compiler/rustc_infer/src/infer/glb.rs
@@ -85,9 +85,9 @@ impl TypeRelation<'tcx> for Glb<'combine, 'infcx, 'tcx> {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
diff --git a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
index e794903fca3..d460222df8a 100644
--- a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
+++ b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
@@ -11,10 +11,10 @@ use rustc_middle::ty::{self, Binder, TypeFoldable};
 impl<'a, 'tcx> CombineFields<'a, 'tcx> {
     pub fn higher_ranked_sub<T>(
         &mut self,
-        a: Binder<T>,
-        b: Binder<T>,
+        a: Binder<'tcx, T>,
+        b: Binder<'tcx, T>,
         a_is_expected: bool,
-    ) -> RelateResult<'tcx, Binder<T>>
+    ) -> RelateResult<'tcx, Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
@@ -50,7 +50,10 @@ impl<'a, 'tcx> CombineFields<'a, 'tcx> {
 
             debug!("higher_ranked_sub: OK result={:?}", result);
 
-            Ok(ty::Binder::bind(result))
+            // We related `a_prime` and `b_prime`, which just had any bound vars
+            // replaced with placeholders or infer vars, respectively. Relating
+            // them should not introduce new bound vars.
+            Ok(ty::Binder::dummy(result))
         })
     }
 }
@@ -66,7 +69,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
     /// the [rustc dev guide].
     ///
     /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/traits/hrtb.html
-    pub fn replace_bound_vars_with_placeholders<T>(&self, binder: ty::Binder<T>) -> T
+    pub fn replace_bound_vars_with_placeholders<T>(&self, binder: ty::Binder<'tcx, T>) -> T
     where
         T: TypeFoldable<'tcx>,
     {
diff --git a/compiler/rustc_infer/src/infer/lub.rs b/compiler/rustc_infer/src/infer/lub.rs
index 9f43fac0916..4fa8f2f1a6a 100644
--- a/compiler/rustc_infer/src/infer/lub.rs
+++ b/compiler/rustc_infer/src/infer/lub.rs
@@ -85,9 +85,9 @@ impl TypeRelation<'tcx> for Lub<'combine, 'infcx, 'tcx> {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index 09eecd715f0..eaec6b46bcd 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -18,7 +18,6 @@ use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::infer::canonical::{Canonical, CanonicalVarValues};
 use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue};
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind, ToType};
-use rustc_middle::mir;
 use rustc_middle::mir::interpret::EvalToConstValueResult;
 use rustc_middle::traits::select;
 use rustc_middle::ty::error::{ExpectedFound, TypeError, UnconstrainedNumeric};
@@ -408,7 +407,7 @@ pub enum SubregionOrigin<'tcx> {
 }
 
 // `SubregionOrigin` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(SubregionOrigin<'_>, 32);
 
 /// Times when we replace late-bound regions with variables:
@@ -1267,15 +1266,6 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
         self.resolve_vars_if_possible(t).to_string()
     }
 
-    pub fn tys_to_string(&self, ts: &[Ty<'tcx>]) -> String {
-        let tstrs: Vec<String> = ts.iter().map(|t| self.ty_to_string(*t)).collect();
-        format!("({})", tstrs.join(", "))
-    }
-
-    pub fn trait_ref_to_string(&self, t: ty::TraitRef<'tcx>) -> String {
-        self.resolve_vars_if_possible(t).print_only_trait_path().to_string()
-    }
-
     /// If `TyVar(vid)` resolves to a type, return that type. Else, return the
     /// universe index of `TyVar(vid)`.
     pub fn probe_ty_var(&self, vid: TyVid) -> Result<Ty<'tcx>, ty::UniverseIndex> {
@@ -1416,7 +1406,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
         &self,
         span: Span,
         lbrct: LateBoundRegionConversionTime,
-        value: ty::Binder<T>,
+        value: ty::Binder<'tcx, T>,
     ) -> (T, BTreeMap<ty::BoundRegion, ty::Region<'tcx>>)
     where
         T: TypeFoldable<'tcx>,
@@ -1499,9 +1489,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
     pub fn const_eval_resolve(
         &self,
         param_env: ty::ParamEnv<'tcx>,
-        def: ty::WithOptConstParam<DefId>,
-        substs: SubstsRef<'tcx>,
-        promoted: Option<mir::Promoted>,
+        ty::Unevaluated { def, substs, promoted }: ty::Unevaluated<'tcx>,
         span: Option<Span>,
     ) -> EvalToConstValueResult<'tcx> {
         let mut original_values = OriginalQueryValues::default();
@@ -1510,7 +1498,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
         let (param_env, substs) = canonical.value;
         // The return value is the evaluated value which doesn't contain any reference to inference
         // variables, thus we don't need to substitute back the original values.
-        self.tcx.const_eval_resolve(param_env, def, substs, promoted, span)
+        self.tcx.const_eval_resolve(param_env, ty::Unevaluated { def, substs, promoted }, span)
     }
 
     /// If `typ` is a type variable of some kind, resolve it one level
@@ -1707,14 +1695,6 @@ impl<'tcx> TypeTrace<'tcx> {
     ) -> TypeTrace<'tcx> {
         TypeTrace { cause: cause.clone(), values: Consts(ExpectedFound::new(a_is_expected, a, b)) }
     }
-
-    pub fn dummy(tcx: TyCtxt<'tcx>) -> TypeTrace<'tcx> {
-        let err = tcx.ty_error();
-        TypeTrace {
-            cause: ObligationCause::dummy(),
-            values: Types(ExpectedFound { expected: err, found: err }),
-        }
-    }
 }
 
 impl<'tcx> SubregionOrigin<'tcx> {
diff --git a/compiler/rustc_infer/src/infer/nll_relate/mod.rs b/compiler/rustc_infer/src/infer/nll_relate/mod.rs
index e5eb771603c..077d2cc20a2 100644
--- a/compiler/rustc_infer/src/infer/nll_relate/mod.rs
+++ b/compiler/rustc_infer/src/infer/nll_relate/mod.rs
@@ -157,7 +157,7 @@ where
 
     fn create_scope(
         &mut self,
-        value: ty::Binder<impl Relate<'tcx>>,
+        value: ty::Binder<'tcx, impl Relate<'tcx>>,
         universally_quantified: UniversallyQuantified,
     ) -> BoundRegionScope<'tcx> {
         let mut scope = BoundRegionScope::default();
@@ -279,7 +279,7 @@ where
     /// Relate a type inference variable with a value type. This works
     /// by creating a "generalization" G of the value where all the
     /// lifetimes are replaced with fresh inference values. This
-    /// genearlization G becomes the value of the inference variable,
+    /// generalization G becomes the value of the inference variable,
     /// and is then related in turn to the value. So e.g. if you had
     /// `vid = ?0` and `value = &'a u32`, we might first instantiate
     /// `?0` to a type like `&'0 u32` where `'0` is a fresh variable,
@@ -608,9 +608,9 @@ where
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
@@ -744,7 +744,7 @@ struct ScopeInstantiator<'me, 'tcx> {
 impl<'me, 'tcx> TypeVisitor<'tcx> for ScopeInstantiator<'me, 'tcx> {
     fn visit_binder<T: TypeFoldable<'tcx>>(
         &mut self,
-        t: &ty::Binder<T>,
+        t: &ty::Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
         self.target_index.shift_in(1);
         t.super_visit_with(self);
@@ -997,9 +997,9 @@ where
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        _: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        _: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
diff --git a/compiler/rustc_infer/src/infer/outlives/env.rs b/compiler/rustc_infer/src/infer/outlives/env.rs
index 1a9e20e79fe..9e04773c5fa 100644
--- a/compiler/rustc_infer/src/infer/outlives/env.rs
+++ b/compiler/rustc_infer/src/infer/outlives/env.rs
@@ -92,11 +92,6 @@ impl<'a, 'tcx> OutlivesEnvironment<'tcx> {
         &self.region_bound_pairs_map
     }
 
-    /// Returns ownership of the `free_region_map`.
-    pub fn into_free_region_map(self) -> FreeRegionMap<'tcx> {
-        self.free_region_map
-    }
-
     /// This is a hack to support the old-skool regionck, which
     /// processes region constraints from the main function and the
     /// closure together. In that context, when we enter a closure, we
diff --git a/compiler/rustc_infer/src/infer/outlives/obligations.rs b/compiler/rustc_infer/src/infer/outlives/obligations.rs
index 16d86e6243d..3e2978fd170 100644
--- a/compiler/rustc_infer/src/infer/outlives/obligations.rs
+++ b/compiler/rustc_infer/src/infer/outlives/obligations.rs
@@ -186,28 +186,6 @@ impl<'cx, 'tcx> InferCtxt<'cx, 'tcx> {
             }
         }
     }
-
-    /// Processes a single ad-hoc region obligation that was not
-    /// registered in advance.
-    pub fn type_must_outlive(
-        &self,
-        region_bound_pairs: &RegionBoundPairs<'tcx>,
-        implicit_region_bound: Option<ty::Region<'tcx>>,
-        param_env: ty::ParamEnv<'tcx>,
-        origin: infer::SubregionOrigin<'tcx>,
-        ty: Ty<'tcx>,
-        region: ty::Region<'tcx>,
-    ) {
-        let outlives = &mut TypeOutlives::new(
-            self,
-            self.tcx,
-            region_bound_pairs,
-            implicit_region_bound,
-            param_env,
-        );
-        let ty = self.resolve_vars_if_possible(ty);
-        outlives.type_must_outlive(origin, ty, region);
-    }
 }
 
 /// The `TypeOutlives` struct has the job of "lowering" a `T: 'a`
diff --git a/compiler/rustc_infer/src/infer/region_constraints/mod.rs b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
index 2902c41a6bc..9ffcddfae99 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/mod.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
@@ -309,31 +309,6 @@ pub struct RegionSnapshot {
     any_unifications: bool,
 }
 
-/// When working with placeholder regions, we often wish to find all of
-/// the regions that are either reachable from a placeholder region, or
-/// which can reach a placeholder region, or both. We call such regions
-/// *tainted* regions. This struct allows you to decide what set of
-/// tainted regions you want.
-#[derive(Debug)]
-pub struct TaintDirections {
-    incoming: bool,
-    outgoing: bool,
-}
-
-impl TaintDirections {
-    pub fn incoming() -> Self {
-        TaintDirections { incoming: true, outgoing: false }
-    }
-
-    pub fn outgoing() -> Self {
-        TaintDirections { incoming: false, outgoing: true }
-    }
-
-    pub fn both() -> Self {
-        TaintDirections { incoming: true, outgoing: true }
-    }
-}
-
 impl<'tcx> RegionConstraintStorage<'tcx> {
     pub fn new() -> Self {
         Self::default()
@@ -472,11 +447,6 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         self.var_infos[vid].universe
     }
 
-    /// Returns the origin for the given variable.
-    pub fn var_origin(&self, vid: RegionVid) -> RegionVariableOrigin {
-        self.var_infos[vid].origin
-    }
-
     fn add_constraint(&mut self, constraint: Constraint<'tcx>, origin: SubregionOrigin<'tcx>) {
         // cannot add constraints once regions are resolved
         debug!("RegionConstraintCollector: add_constraint({:?})", constraint);
@@ -795,16 +765,6 @@ impl<'tcx> VerifyBound<'tcx> {
             VerifyBound::AnyBound(vec![self, vb])
         }
     }
-
-    pub fn and(self, vb: VerifyBound<'tcx>) -> VerifyBound<'tcx> {
-        if self.must_hold() && vb.must_hold() {
-            self
-        } else if self.cannot_hold() && vb.cannot_hold() {
-            self
-        } else {
-            VerifyBound::AllBounds(vec![self, vb])
-        }
-    }
 }
 
 impl<'tcx> RegionConstraintData<'tcx> {
diff --git a/compiler/rustc_infer/src/infer/resolve.rs b/compiler/rustc_infer/src/infer/resolve.rs
index d72be0134fb..48b8ee17594 100644
--- a/compiler/rustc_infer/src/infer/resolve.rs
+++ b/compiler/rustc_infer/src/infer/resolve.rs
@@ -1,5 +1,6 @@
 use super::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use super::{FixupError, FixupResult, InferCtxt, Span};
+use rustc_middle::mir;
 use rustc_middle::ty::fold::{TypeFolder, TypeVisitor};
 use rustc_middle::ty::{self, Const, InferConst, Ty, TyCtxt, TypeFoldable};
 
@@ -46,6 +47,10 @@ impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticVarResolver<'a, 'tcx> {
             ct.super_fold_with(self)
         }
     }
+
+    fn fold_mir_const(&mut self, constant: mir::ConstantKind<'tcx>) -> mir::ConstantKind<'tcx> {
+        constant.super_fold_with(self)
+    }
 }
 
 /// The opportunistic region resolver opportunistically resolves regions
diff --git a/compiler/rustc_infer/src/infer/sub.rs b/compiler/rustc_infer/src/infer/sub.rs
index 66871985158..bf5f328233d 100644
--- a/compiler/rustc_infer/src/infer/sub.rs
+++ b/compiler/rustc_infer/src/infer/sub.rs
@@ -162,9 +162,9 @@ impl TypeRelation<'tcx> for Sub<'combine, 'infcx, 'tcx> {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs
index 35b97fff3da..683c1df783e 100644
--- a/compiler/rustc_infer/src/infer/type_variable.rs
+++ b/compiler/rustc_infer/src/infer/type_variable.rs
@@ -146,9 +146,7 @@ impl<'tcx> TypeVariableValue<'tcx> {
     }
 }
 
-pub(crate) struct Instantiate {
-    vid: ty::TyVid,
-}
+pub(crate) struct Instantiate;
 
 pub(crate) struct Delegate;
 
@@ -224,7 +222,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
         // Hack: we only need this so that `types_escaping_snapshot`
         // can see what has been unified; see the Delegate impl for
         // more details.
-        self.undo_log.push(Instantiate { vid });
+        self.undo_log.push(Instantiate);
     }
 
     /// Creates a new type variable.
@@ -346,56 +344,6 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
         )
     }
 
-    /// Finds the set of type variables that existed *before* `s`
-    /// but which have only been unified since `s` started, and
-    /// return the types with which they were unified. So if we had
-    /// a type variable `V0`, then we started the snapshot, then we
-    /// created a type variable `V1`, unified `V0` with `T0`, and
-    /// unified `V1` with `T1`, this function would return `{T0}`.
-    pub fn types_escaping_snapshot(&mut self, s: &super::Snapshot<'tcx>) -> Vec<Ty<'tcx>> {
-        let mut new_elem_threshold = u32::MAX;
-        let mut escaping_types = Vec::new();
-        let actions_since_snapshot = self.undo_log.actions_since_snapshot(s);
-        debug!("actions_since_snapshot.len() = {}", actions_since_snapshot.len());
-        for i in 0..actions_since_snapshot.len() {
-            let actions_since_snapshot = self.undo_log.actions_since_snapshot(s);
-            match actions_since_snapshot[i] {
-                super::UndoLog::TypeVariables(UndoLog::Values(sv::UndoLog::NewElem(index))) => {
-                    // if any new variables were created during the
-                    // snapshot, remember the lower index (which will
-                    // always be the first one we see). Note that this
-                    // action must precede those variables being
-                    // specified.
-                    new_elem_threshold = cmp::min(new_elem_threshold, index as u32);
-                    debug!("NewElem({}) new_elem_threshold={}", index, new_elem_threshold);
-                }
-
-                super::UndoLog::TypeVariables(UndoLog::Values(sv::UndoLog::Other(
-                    Instantiate { vid, .. },
-                ))) => {
-                    if vid.index < new_elem_threshold {
-                        // quick check to see if this variable was
-                        // created since the snapshot started or not.
-                        let mut eq_relations = ut::UnificationTable::with_log(
-                            &mut self.storage.eq_relations,
-                            &mut *self.undo_log,
-                        );
-                        let escaping_type = match eq_relations.probe_value(vid) {
-                            TypeVariableValue::Unknown { .. } => bug!(),
-                            TypeVariableValue::Known { value } => value,
-                        };
-                        escaping_types.push(escaping_type);
-                    }
-                    debug!("SpecifyVar({:?}) new_elem_threshold={}", vid, new_elem_threshold);
-                }
-
-                _ => {}
-            }
-        }
-
-        escaping_types
-    }
-
     /// Returns indices of all variables that are not yet
     /// instantiated.
     pub fn unsolved_variables(&mut self) -> Vec<ty::TyVid> {
diff --git a/compiler/rustc_infer/src/infer/undo_log.rs b/compiler/rustc_infer/src/infer/undo_log.rs
index 4be0e7948f7..f41e872e004 100644
--- a/compiler/rustc_infer/src/infer/undo_log.rs
+++ b/compiler/rustc_infer/src/infer/undo_log.rs
@@ -165,10 +165,6 @@ impl<'tcx> InferCtxtInner<'tcx> {
 }
 
 impl<'tcx> InferCtxtUndoLogs<'tcx> {
-    pub fn actions_since_snapshot(&self, snapshot: &Snapshot<'tcx>) -> &[UndoLog<'tcx>] {
-        &self.logs[snapshot.undo_len..]
-    }
-
     pub fn start_snapshot(&mut self) -> Snapshot<'tcx> {
         self.num_open_snapshots += 1;
         Snapshot { undo_len: self.logs.len(), _marker: PhantomData }
diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs
index 3690a88c0d9..25a262d7e48 100644
--- a/compiler/rustc_infer/src/lib.rs
+++ b/compiler/rustc_infer/src/lib.rs
@@ -2,7 +2,7 @@
 //!
 //! - **Type inference.** The type inference code can be found in the `infer` module;
 //!   this code handles low-level equality and subtyping operations. The
-//!   type check pass in the compiler is found in the `librustc_typeck` crate.
+//!   type check pass in the compiler is found in the `rustc_typeck` crate.
 //!
 //! For more information about how rustc works, see the [rustc dev guide].
 //!
@@ -19,15 +19,16 @@
 #![feature(const_fn)]
 #![feature(const_panic)]
 #![feature(extend_one)]
+#![feature(iter_zip)]
 #![feature(never_type)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(in_band_lifetimes)]
 #![feature(control_flow_enum)]
 #![recursion_limit = "512"] // For rustdoc
 
 #[macro_use]
 extern crate rustc_macros;
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 #[macro_use]
 extern crate rustc_data_structures;
 #[macro_use]
diff --git a/compiler/rustc_infer/src/traits/error_reporting/mod.rs b/compiler/rustc_infer/src/traits/error_reporting/mod.rs
index 835f75ec8ef..0ac4b6b25bb 100644
--- a/compiler/rustc_infer/src/traits/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/traits/error_reporting/mod.rs
@@ -9,6 +9,7 @@ use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::Symbol;
 use rustc_span::{MultiSpan, Span};
 use std::fmt;
+use std::iter;
 
 impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
     pub fn report_extra_impl_obligation(
@@ -94,7 +95,7 @@ pub fn report_object_safety_error(
         note_span
             .push_span_label(trait_span, "this trait cannot be made into an object...".to_string());
     }
-    for (span, msg) in multi_span.into_iter().zip(messages.into_iter()) {
+    for (span, msg) in iter::zip(multi_span, messages) {
         note_span.push_span_label(span, msg);
     }
     err.span_note(
@@ -104,7 +105,7 @@ pub fn report_object_safety_error(
          <https://doc.rust-lang.org/reference/items/traits.html#object-safety>",
     );
 
-    if tcx.sess.trait_methods_not_found.borrow().contains(&span) {
+    if tcx.sess.trait_methods_not_found.borrow().iter().any(|full_span| full_span.contains(span)) {
         // Avoid emitting error caused by non-existing method (#58734)
         err.cancel();
     }
diff --git a/compiler/rustc_infer/src/traits/mod.rs b/compiler/rustc_infer/src/traits/mod.rs
index aaf5e958c26..a33234a91fa 100644
--- a/compiler/rustc_infer/src/traits/mod.rs
+++ b/compiler/rustc_infer/src/traits/mod.rs
@@ -56,7 +56,7 @@ pub type PredicateObligation<'tcx> = Obligation<'tcx, ty::Predicate<'tcx>>;
 pub type TraitObligation<'tcx> = Obligation<'tcx, ty::PolyTraitPredicate<'tcx>>;
 
 // `PredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(PredicateObligation<'_>, 32);
 
 pub type PredicateObligations<'tcx> = Vec<PredicateObligation<'tcx>>;
@@ -128,7 +128,7 @@ impl<'tcx> FulfillmentError<'tcx> {
 }
 
 impl<'tcx> TraitObligation<'tcx> {
-    pub fn self_ty(&self) -> ty::Binder<Ty<'tcx>> {
+    pub fn self_ty(&self) -> ty::Binder<'tcx, Ty<'tcx>> {
         self.predicate.map_bound(|p| p.self_ty())
     }
 }
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index cfe98a630c1..3bfe8da505f 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -10,8 +10,8 @@ doctest = false
 [dependencies]
 libc = "0.2"
 tracing = "0.1"
-rustc-rayon-core = "0.3.0"
-rayon = { version = "0.3.0", package = "rustc-rayon" }
+rustc-rayon-core = "0.3.1"
+rayon = { version = "0.3.1", package = "rustc-rayon" }
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
diff --git a/compiler/rustc_interface/src/callbacks.rs b/compiler/rustc_interface/src/callbacks.rs
index 7fa1a3eb0f5..3b852b8ccf9 100644
--- a/compiler/rustc_interface/src/callbacks.rs
+++ b/compiler/rustc_interface/src/callbacks.rs
@@ -1,6 +1,6 @@
 //! Throughout the compiler tree, there are several places which want to have
 //! access to state or queries while being inside crates that are dependencies
-//! of librustc_middle. To facilitate this, we have the
+//! of `rustc_middle`. To facilitate this, we have the
 //! `rustc_data_structures::AtomicRef` type, which allows us to setup a global
 //! static which can then be set in this file at program startup.
 //!
@@ -13,8 +13,8 @@ use rustc_errors::{Diagnostic, TRACK_DIAGNOSTICS};
 use rustc_middle::ty::tls;
 use std::fmt;
 
-/// This is a callback from librustc_ast as it cannot access the implicit state
-/// in librustc_middle otherwise.
+/// This is a callback from `rustc_ast` as it cannot access the implicit state
+/// in `rustc_middle` otherwise.
 fn span_debug(span: rustc_span::Span, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     tls::with_opt(|tcx| {
         if let Some(tcx) = tcx {
@@ -25,8 +25,8 @@ fn span_debug(span: rustc_span::Span, f: &mut fmt::Formatter<'_>) -> fmt::Result
     })
 }
 
-/// This is a callback from librustc_ast as it cannot access the implicit state
-/// in librustc_middle otherwise. It is used to when diagnostic messages are
+/// This is a callback from `rustc_ast` as it cannot access the implicit state
+/// in `rustc_middle` otherwise. It is used to when diagnostic messages are
 /// emitted and stores them in the current query, if there is one.
 fn track_diagnostic(diagnostic: &Diagnostic) {
     tls::with_context_opt(|icx| {
@@ -39,8 +39,8 @@ fn track_diagnostic(diagnostic: &Diagnostic) {
     })
 }
 
-/// This is a callback from librustc_hir as it cannot access the implicit state
-/// in librustc_middle otherwise.
+/// This is a callback from `rustc_hir` as it cannot access the implicit state
+/// in `rustc_middle` otherwise.
 fn def_id_debug(def_id: rustc_hir::def_id::DefId, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     write!(f, "DefId({}:{}", def_id.krate, def_id.index.index())?;
     tls::with_opt(|opt_tcx| {
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 502e7155c2e..a1090ee316d 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -142,6 +142,9 @@ pub struct Config {
 
     pub lint_caps: FxHashMap<lint::LintId, lint::Level>,
 
+    /// This is a callback from the driver that is called when [`ParseSess`] is created.
+    pub parse_sess_created: Option<Box<dyn FnOnce(&mut ParseSess) + Send>>,
+
     /// This is a callback from the driver that is called when we're registering lints;
     /// it is called during plugin registration when we have the LintStore in a non-shared state.
     ///
@@ -166,7 +169,7 @@ pub struct Config {
 
 pub fn create_compiler_and_run<R>(config: Config, f: impl FnOnce(&Compiler) -> R) -> R {
     let registry = &config.registry;
-    let (sess, codegen_backend) = util::create_session(
+    let (mut sess, codegen_backend) = util::create_session(
         config.opts,
         config.crate_cfg,
         config.diagnostic_output,
@@ -177,6 +180,14 @@ pub fn create_compiler_and_run<R>(config: Config, f: impl FnOnce(&Compiler) -> R
         registry.clone(),
     );
 
+    if let Some(parse_sess_created) = config.parse_sess_created {
+        parse_sess_created(
+            &mut Lrc::get_mut(&mut sess)
+                .expect("create_session() should never share the returned session")
+                .parse_sess,
+        );
+    }
+
     let compiler = Compiler {
         sess,
         codegen_backend,
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 5217066bbef..02e62a2cee9 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -15,8 +15,8 @@ use rustc_expand::base::ExtCtxt;
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_hir::Crate;
-use rustc_index::vec::IndexVec;
 use rustc_lint::LintStore;
+use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
 use rustc_middle::middle;
@@ -302,8 +302,10 @@ fn configure_and_expand_inner<'a>(
             ..rustc_expand::expand::ExpansionConfig::default(crate_name.to_string())
         };
 
-        let extern_mod_loaded = |k: &ast::Crate, ident: Ident| {
-            pre_expansion_lint(sess, lint_store, k, &*ident.name.as_str())
+        let extern_mod_loaded = |ident: Ident, attrs, items, span| {
+            let krate = ast::Crate { attrs, items, span, proc_macros: vec![] };
+            pre_expansion_lint(sess, lint_store, &krate, &ident.name.as_str());
+            (krate.attrs, krate.items)
         };
         let mut ecx = ExtCtxt::new(&sess, cfg, &mut resolver, Some(&extern_mod_loaded));
 
@@ -786,13 +788,7 @@ pub fn create_global_ctxt<'tcx>(
         callback(sess, &mut local_providers, &mut extern_providers);
     }
 
-    let queries = {
-        let crates = resolver_outputs.cstore.crates_untracked();
-        let max_cnum = crates.iter().map(|c| c.as_usize()).max().unwrap_or(0);
-        let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1);
-        providers[LOCAL_CRATE] = local_providers;
-        queries.get_or_init(|| TcxQueries::new(providers, extern_providers))
-    };
+    let queries = queries.get_or_init(|| TcxQueries::new(local_providers, extern_providers));
 
     let gcx = sess.time("setup_global_ctxt", || {
         global_ctxt.get_or_init(|| {
@@ -836,6 +832,12 @@ fn analysis(tcx: TyCtxt<'_>, cnum: CrateNum) -> Result<()> {
                 });
 
                 sess.time("looking_for_derive_registrar", || proc_macro_decls::find(tcx));
+
+                let cstore = tcx
+                    .cstore_as_any()
+                    .downcast_ref::<CStore>()
+                    .expect("`tcx.cstore` is not a `CStore`");
+                cstore.report_unused_deps(tcx);
             },
             {
                 par_iter(&tcx.hir().krate().modules).for_each(|(&module, _)| {
@@ -988,7 +990,7 @@ fn encode_and_write_metadata(
             .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err)));
         let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps);
         let metadata_filename = emit_metadata(tcx.sess, &metadata, &metadata_tmpdir);
-        if let Err(e) = fs::rename(&metadata_filename, &out_filename) {
+        if let Err(e) = util::non_durable_rename(&metadata_filename, &out_filename) {
             tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
         }
         if tcx.sess.opts.json_artifact_notifications {
@@ -1026,9 +1028,6 @@ pub fn start_codegen<'tcx>(
         rustc_symbol_mangling::test::report_symbol_names(tcx);
     }
 
-    tcx.sess.time("assert_dep_graph", || rustc_incremental::assert_dep_graph(tcx));
-    tcx.sess.time("serialize_dep_graph", || rustc_incremental::save_dep_graph(tcx));
-
     info!("Post-codegen\n{:?}", tcx.debug_stats());
 
     if tcx.sess.opts.output_types.contains_key(&OutputType::Mir) {
diff --git a/compiler/rustc_interface/src/proc_macro_decls.rs b/compiler/rustc_interface/src/proc_macro_decls.rs
index d0262935c89..4637055a82d 100644
--- a/compiler/rustc_interface/src/proc_macro_decls.rs
+++ b/compiler/rustc_interface/src/proc_macro_decls.rs
@@ -25,7 +25,8 @@ struct Finder<'tcx> {
 
 impl<'v> ItemLikeVisitor<'v> for Finder<'_> {
     fn visit_item(&mut self, item: &hir::Item<'_>) {
-        if self.tcx.sess.contains_name(&item.attrs, sym::rustc_proc_macro_decls) {
+        let attrs = self.tcx.hir().attrs(item.hir_id());
+        if self.tcx.sess.contains_name(attrs, sym::rustc_proc_macro_decls) {
             self.decls = Some(item.hir_id());
         }
     }
diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs
index 9c38d2b91ab..01853eab530 100644
--- a/compiler/rustc_interface/src/queries.rs
+++ b/compiler/rustc_interface/src/queries.rs
@@ -207,7 +207,13 @@ impl<'tcx> Queries<'tcx> {
                                 })
                                 .open(self.session())
                         });
-                    DepGraph::new(prev_graph, prev_work_products)
+
+                    rustc_incremental::build_dep_graph(
+                        self.session(),
+                        prev_graph,
+                        prev_work_products,
+                    )
+                    .unwrap_or_else(DepGraph::new_disabled)
                 }
             })
         })
@@ -435,6 +441,9 @@ impl Compiler {
             if self.session().opts.debugging_opts.query_stats {
                 gcx.enter(rustc_query_impl::print_stats);
             }
+
+            self.session()
+                .time("serialize_dep_graph", || gcx.enter(rustc_incremental::save_dep_graph));
         }
 
         _timer = Some(self.session().timer("free_global_ctxt"));
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 9a11b534887..9685d21762b 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -2,12 +2,13 @@ use crate::interface::parse_cfgspecs;
 
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
+use rustc_session::config::InstrumentCoverage;
 use rustc_session::config::Strip;
 use rustc_session::config::{build_configuration, build_session_options, to_crate_config};
 use rustc_session::config::{rustc_optgroups, ErrorOutputType, ExternLocation, Options, Passes};
 use rustc_session::config::{CFGuard, ExternEntry, LinkerPluginLto, LtoCli, SwitchWithOptPath};
 use rustc_session::config::{
-    Externs, OutputType, OutputTypes, SanitizerSet, SymbolManglingVersion, WasiExecModel,
+    Externs, OutputType, OutputTypes, SymbolManglingVersion, WasiExecModel,
 };
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
@@ -17,7 +18,7 @@ use rustc_span::edition::{Edition, DEFAULT_EDITION};
 use rustc_span::symbol::sym;
 use rustc_span::SourceFileHashAlgorithm;
 use rustc_target::spec::{CodeModel, LinkerFlavor, MergeFunctions, PanicStrategy};
-use rustc_target::spec::{RelocModel, RelroLevel, SplitDebuginfo, TlsModel};
+use rustc_target::spec::{RelocModel, RelroLevel, SanitizerSet, SplitDebuginfo, TlsModel};
 use std::collections::{BTreeMap, BTreeSet};
 use std::iter::FromIterator;
 use std::num::NonZeroUsize;
@@ -404,7 +405,6 @@ fn test_codegen_options_tracking_hash() {
     untracked!(incremental, Some(String::from("abc")));
     // `link_arg` is omitted because it just forwards to `link_args`.
     untracked!(link_args, vec![String::from("abc"), String::from("def")]);
-    untracked!(link_dead_code, Some(true));
     untracked!(link_self_contained, Some(true));
     untracked!(linker, Some(PathBuf::from("linker")));
     untracked!(linker_flavor, Some(LinkerFlavor::Gcc));
@@ -432,6 +432,7 @@ fn test_codegen_options_tracking_hash() {
     tracked!(force_unwind_tables, Some(true));
     tracked!(inline_threshold, Some(0xf007ba11));
     tracked!(linker_plugin_lto, LinkerPluginLto::LinkerPluginAuto);
+    tracked!(link_dead_code, Some(true));
     tracked!(llvm_args, vec![String::from("1"), String::from("2")]);
     tracked!(lto, LtoCli::Fat);
     tracked!(metadata, vec![String::from("A"), String::from("B")]);
@@ -470,7 +471,6 @@ fn test_debugging_options_tracking_hash() {
     untracked!(ast_json, true);
     untracked!(ast_json_noexpand, true);
     untracked!(borrowck, String::from("other"));
-    untracked!(borrowck_stats, true);
     untracked!(deduplicate_diagnostics, true);
     untracked!(dep_tasks, true);
     untracked!(dont_buffer_diagnostics, true);
@@ -560,14 +560,13 @@ fn test_debugging_options_tracking_hash() {
     tracked!(inline_mir, Some(true));
     tracked!(inline_mir_threshold, Some(123));
     tracked!(inline_mir_hint_threshold, Some(123));
-    tracked!(insert_sideeffect, true);
-    tracked!(instrument_coverage, true);
+    tracked!(instrument_coverage, Some(InstrumentCoverage::All));
     tracked!(instrument_mcount, true);
     tracked!(link_only, true);
     tracked!(merge_functions, Some(MergeFunctions::Disabled));
     tracked!(mir_emit_retag, true);
-    tracked!(mir_opt_level, 3);
-    tracked!(mutable_noalias, true);
+    tracked!(mir_opt_level, Some(4));
+    tracked!(mutable_noalias, Some(true));
     tracked!(new_llvm_pass_manager, true);
     tracked!(no_codegen, true);
     tracked!(no_generate_arange_section, true);
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 798996263c7..59488fc80a5 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -265,7 +265,7 @@ pub fn get_codegen_backend(sopts: &config::Options) -> Box<dyn CodegenBackend> {
 
         let backend = match codegen_name {
             filename if filename.contains('.') => load_backend_from_dylib(filename.as_ref()),
-            codegen_name => get_builtin_codegen_backend(codegen_name),
+            codegen_name => get_builtin_codegen_backend(&sopts.maybe_sysroot, codegen_name),
         };
 
         unsafe {
@@ -390,15 +390,21 @@ fn sysroot_candidates() -> Vec<PathBuf> {
     }
 }
 
-pub fn get_builtin_codegen_backend(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
+pub fn get_builtin_codegen_backend(
+    maybe_sysroot: &Option<PathBuf>,
+    backend_name: &str,
+) -> fn() -> Box<dyn CodegenBackend> {
     match backend_name {
         #[cfg(feature = "llvm")]
         "llvm" => rustc_codegen_llvm::LlvmCodegenBackend::new,
-        _ => get_codegen_sysroot(backend_name),
+        _ => get_codegen_sysroot(maybe_sysroot, backend_name),
     }
 }
 
-pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
+pub fn get_codegen_sysroot(
+    maybe_sysroot: &Option<PathBuf>,
+    backend_name: &str,
+) -> fn() -> Box<dyn CodegenBackend> {
     // For now we only allow this function to be called once as it'll dlopen a
     // few things, which seems to work best if we only do that once. In
     // general this assertion never trips due to the once guard in `get_codegen_backend`,
@@ -413,8 +419,9 @@ pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend
     let target = session::config::host_triple();
     let sysroot_candidates = sysroot_candidates();
 
-    let sysroot = sysroot_candidates
+    let sysroot = maybe_sysroot
         .iter()
+        .chain(sysroot_candidates.iter())
         .map(|sysroot| {
             let libdir = filesearch::relative_target_lib_path(&sysroot, &target);
             sysroot.join(libdir).with_file_name("codegen-backends")
@@ -450,8 +457,10 @@ pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend
 
     let mut file: Option<PathBuf> = None;
 
-    let expected_name =
-        format!("rustc_codegen_{}-{}", backend_name, release_str().expect("CFG_RELEASE"));
+    let expected_names = &[
+        format!("rustc_codegen_{}-{}", backend_name, release_str().expect("CFG_RELEASE")),
+        format!("rustc_codegen_{}", backend_name),
+    ];
     for entry in d.filter_map(|e| e.ok()) {
         let path = entry.path();
         let filename = match path.file_name().and_then(|s| s.to_str()) {
@@ -462,7 +471,7 @@ pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend
             continue;
         }
         let name = &filename[DLL_PREFIX.len()..filename.len() - DLL_SUFFIX.len()];
-        if name != expected_name {
+        if !expected_names.iter().any(|expected| expected == name) {
             continue;
         }
         if let Some(ref prev) = file {
@@ -694,16 +703,42 @@ pub fn build_output_filenames(
     }
 }
 
-// Note: Also used by librustdoc, see PR #43348. Consider moving this struct elsewhere.
-//
-// FIXME: Currently the `everybody_loops` transformation is not applied to:
-//  * `const fn`, due to issue #43636 that `loop` is not supported for const evaluation. We are
-//    waiting for miri to fix that.
-//  * `impl Trait`, due to issue #43869 that functions returning impl Trait cannot be diverging.
-//    Solving this may require `!` to implement every trait, which relies on the an even more
-//    ambitious form of the closed RFC #1637. See also [#34511].
-//
-// [#34511]: https://github.com/rust-lang/rust/issues/34511#issuecomment-322340401
+#[cfg(not(target_os = "linux"))]
+pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
+    std::fs::rename(src, dst)
+}
+
+/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems
+/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully"
+/// write back the source file before committing the rename in case a developer forgot some of
+/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates.
+///
+/// To avoid triggering this heuristic we delete the destination first, if it exists.
+/// The cost of an extra syscall is much lower than getting descheduled for the sync IO.
+#[cfg(target_os = "linux")]
+pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
+    let _ = std::fs::remove_file(dst);
+    std::fs::rename(src, dst)
+}
+
+/// Replaces function bodies with `loop {}` (an infinite loop). This gets rid of
+/// all semantic errors in the body while still satisfying the return type,
+/// except in certain cases, see below for more.
+///
+/// This pass is known as `everybody_loops`. Very punny.
+///
+/// As of March 2021, `everybody_loops` is only used for the
+/// `-Z unpretty=everybody_loops` debugging option.
+///
+/// FIXME: Currently the `everybody_loops` transformation is not applied to:
+///  * `const fn`; support could be added, but hasn't. Originally `const fn`
+///    was skipped due to issue #43636 that `loop` was not supported for
+///    const evaluation.
+///  * `impl Trait`, due to issue #43869 that functions returning impl Trait cannot be diverging.
+///    Solving this may require `!` to implement every trait, which relies on the an even more
+///    ambitious form of the closed RFC #1637. See also [#34511].
+///
+/// [#34511]: https://github.com/rust-lang/rust/issues/34511#issuecomment-322340401
 pub struct ReplaceBodyWithLoop<'a, 'b> {
     within_static_or_const: bool,
     nested_blocks: Option<Vec<ast::Block>>,
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index 44fc4db7dc1..b9781581ff7 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -1,13 +1,13 @@
 //! Low-level Rust lexer.
 //!
-//! The idea with `librustc_lexer` is to make a reusable library,
+//! The idea with `rustc_lexer` is to make a reusable library,
 //! by separating out pure lexing and rustc-specific concerns, like spans,
 //! error reporting, and interning.  So, rustc_lexer operates directly on `&str`,
 //! produces simple tokens which are a pair of type-tag and a bit of original text,
 //! and does not report errors, instead storing them as flags on the token.
 //!
 //! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
-//! For that see [`librustc_parse::lexer`], which converts this basic token stream
+//! For that see [`rustc_parse::lexer`], which converts this basic token stream
 //! into wide tokens used by actual parser.
 //!
 //! The purpose of this crate is to convert raw sources into a labeled sequence
@@ -17,7 +17,7 @@
 //! The main entity of this crate is the [`TokenKind`] enum which represents common
 //! lexeme types.
 //!
-//! [`librustc_parse::lexer`]: ../rustc_parse/lexer/index.html
+//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
 // We want to be able to build this crate with a stable compiler, so no
 // `#![feature]` attributes should be added.
 
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index 4258a4b4237..3965a3dcdfd 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -1,3 +1,5 @@
+// ignore-tidy-filelength
+
 //! Lints in the Rust compiler.
 //!
 //! This contains lints which can feasibly be implemented as their own
@@ -481,7 +483,7 @@ fn has_doc(sess: &Session, attr: &ast::Attribute) -> bool {
         return false;
     }
 
-    if attr.is_value_str() {
+    if attr.value_str().is_some() {
         return true;
     }
 
@@ -508,8 +510,7 @@ impl MissingDoc {
     fn check_missing_docs_attrs(
         &self,
         cx: &LateContext<'_>,
-        id: Option<hir::HirId>,
-        attrs: &[ast::Attribute],
+        id: hir::HirId,
         sp: Span,
         article: &'static str,
         desc: &'static str,
@@ -528,12 +529,13 @@ impl MissingDoc {
         // Only check publicly-visible items, using the result from the privacy pass.
         // It's an option so the crate root can also use this function (it doesn't
         // have a `NodeId`).
-        if let Some(id) = id {
+        if id != hir::CRATE_HIR_ID {
             if !cx.access_levels.is_exported(id) {
                 return;
             }
         }
 
+        let attrs = cx.tcx.hir().attrs(id);
         let has_doc = attrs.iter().any(|a| has_doc(cx.sess(), a));
         if !has_doc {
             cx.struct_span_lint(
@@ -565,10 +567,11 @@ impl<'tcx> LateLintPass<'tcx> for MissingDoc {
     }
 
     fn check_crate(&mut self, cx: &LateContext<'_>, krate: &hir::Crate<'_>) {
-        self.check_missing_docs_attrs(cx, None, &krate.item.attrs, krate.item.span, "the", "crate");
+        self.check_missing_docs_attrs(cx, hir::CRATE_HIR_ID, krate.item.inner, "the", "crate");
 
         for macro_def in krate.exported_macros {
-            let has_doc = macro_def.attrs.iter().any(|a| has_doc(cx.sess(), a));
+            let attrs = cx.tcx.hir().attrs(macro_def.hir_id());
+            let has_doc = attrs.iter().any(|a| has_doc(cx.sess(), a));
             if !has_doc {
                 cx.struct_span_lint(
                     MISSING_DOCS,
@@ -622,7 +625,7 @@ impl<'tcx> LateLintPass<'tcx> for MissingDoc {
 
         let (article, desc) = cx.tcx.article_and_description(it.def_id.to_def_id());
 
-        self.check_missing_docs_attrs(cx, Some(it.hir_id()), &it.attrs, it.span, article, desc);
+        self.check_missing_docs_attrs(cx, it.hir_id(), it.span, article, desc);
     }
 
     fn check_trait_item(&mut self, cx: &LateContext<'_>, trait_item: &hir::TraitItem<'_>) {
@@ -632,14 +635,7 @@ impl<'tcx> LateLintPass<'tcx> for MissingDoc {
 
         let (article, desc) = cx.tcx.article_and_description(trait_item.def_id.to_def_id());
 
-        self.check_missing_docs_attrs(
-            cx,
-            Some(trait_item.hir_id()),
-            &trait_item.attrs,
-            trait_item.span,
-            article,
-            desc,
-        );
+        self.check_missing_docs_attrs(cx, trait_item.hir_id(), trait_item.span, article, desc);
     }
 
     fn check_impl_item(&mut self, cx: &LateContext<'_>, impl_item: &hir::ImplItem<'_>) {
@@ -649,43 +645,22 @@ impl<'tcx> LateLintPass<'tcx> for MissingDoc {
         }
 
         let (article, desc) = cx.tcx.article_and_description(impl_item.def_id.to_def_id());
-        self.check_missing_docs_attrs(
-            cx,
-            Some(impl_item.hir_id()),
-            &impl_item.attrs,
-            impl_item.span,
-            article,
-            desc,
-        );
+        self.check_missing_docs_attrs(cx, impl_item.hir_id(), impl_item.span, article, desc);
     }
 
     fn check_foreign_item(&mut self, cx: &LateContext<'_>, foreign_item: &hir::ForeignItem<'_>) {
         let (article, desc) = cx.tcx.article_and_description(foreign_item.def_id.to_def_id());
-        self.check_missing_docs_attrs(
-            cx,
-            Some(foreign_item.hir_id()),
-            &foreign_item.attrs,
-            foreign_item.span,
-            article,
-            desc,
-        );
+        self.check_missing_docs_attrs(cx, foreign_item.hir_id(), foreign_item.span, article, desc);
     }
 
-    fn check_struct_field(&mut self, cx: &LateContext<'_>, sf: &hir::StructField<'_>) {
+    fn check_field_def(&mut self, cx: &LateContext<'_>, sf: &hir::FieldDef<'_>) {
         if !sf.is_positional() {
-            self.check_missing_docs_attrs(
-                cx,
-                Some(sf.hir_id),
-                &sf.attrs,
-                sf.span,
-                "a",
-                "struct field",
-            )
+            self.check_missing_docs_attrs(cx, sf.hir_id, sf.span, "a", "struct field")
         }
     }
 
     fn check_variant(&mut self, cx: &LateContext<'_>, v: &hir::Variant<'_>) {
-        self.check_missing_docs_attrs(cx, Some(v.id), &v.attrs, v.span, "a", "variant");
+        self.check_missing_docs_attrs(cx, v.id, v.span, "a", "variant");
     }
 }
 
@@ -884,11 +859,10 @@ declare_lint! {
     /// ```
     ///
     /// This syntax is now a hard error in the 2018 edition. In the 2015
-    /// edition, this lint is "allow" by default, because the old code is
-    /// still valid, and warning for all old code can be noisy. This lint
+    /// edition, this lint is "warn" by default. This lint
     /// enables the [`cargo fix`] tool with the `--edition` flag to
     /// automatically transition old code from the 2015 edition to 2018. The
-    /// tool will switch this lint to "warn" and will automatically apply the
+    /// tool will run this lint and automatically apply the
     /// suggested fix from the compiler (which is to add `_` to each
     /// parameter). This provides a completely automated way to update old
     /// code for a new edition. See [issue #41686] for more details.
@@ -896,7 +870,7 @@ declare_lint! {
     /// [issue #41686]: https://github.com/rust-lang/rust/issues/41686
     /// [`cargo fix`]: https://doc.rust-lang.org/cargo/commands/cargo-fix.html
     pub ANONYMOUS_PARAMETERS,
-    Allow,
+    Warn,
     "detects anonymous parameters",
     @future_incompatible = FutureIncompatibleInfo {
         reference: "issue #41686 <https://github.com/rust-lang/rust/issues/41686>",
@@ -911,6 +885,10 @@ declare_lint_pass!(
 
 impl EarlyLintPass for AnonymousParameters {
     fn check_trait_item(&mut self, cx: &EarlyContext<'_>, it: &ast::AssocItem) {
+        if cx.sess.edition() != Edition::Edition2015 {
+            // This is a hard error in future editions; avoid linting and erroring
+            return;
+        }
         if let ast::AssocItemKind::Fn(box FnKind(_, ref sig, _, _)) = it.kind {
             for arg in sig.decl.inputs.iter() {
                 if let ast::PatKind::Ident(_, ident, None) = arg.pat.kind {
@@ -1016,7 +994,7 @@ fn warn_if_doc(cx: &EarlyContext<'_>, node_span: Span, node_kind: &str, attrs: &
                 Some(sugared_span.map_or(attr.span, |span| span.with_hi(attr.span.hi())));
         }
 
-        if attrs.peek().map(|next_attr| next_attr.is_doc_comment()).unwrap_or_default() {
+        if attrs.peek().map_or(false, |next_attr| next_attr.is_doc_comment()) {
             continue;
         }
 
@@ -1119,9 +1097,10 @@ declare_lint_pass!(InvalidNoMangleItems => [NO_MANGLE_CONST_ITEMS, NO_MANGLE_GEN
 
 impl<'tcx> LateLintPass<'tcx> for InvalidNoMangleItems {
     fn check_item(&mut self, cx: &LateContext<'_>, it: &hir::Item<'_>) {
+        let attrs = cx.tcx.hir().attrs(it.hir_id());
         match it.kind {
             hir::ItemKind::Fn(.., ref generics, _) => {
-                if let Some(no_mangle_attr) = cx.sess().find_by_name(&it.attrs, sym::no_mangle) {
+                if let Some(no_mangle_attr) = cx.sess().find_by_name(attrs, sym::no_mangle) {
                     for param in generics.params {
                         match param.kind {
                             GenericParamKind::Lifetime { .. } => {}
@@ -1147,7 +1126,7 @@ impl<'tcx> LateLintPass<'tcx> for InvalidNoMangleItems {
                 }
             }
             hir::ItemKind::Const(..) => {
-                if cx.sess().contains_name(&it.attrs, sym::no_mangle) {
+                if cx.sess().contains_name(attrs, sym::no_mangle) {
                     // Const items do not refer to a particular location in memory, and therefore
                     // don't have anything to attach a symbol to
                     cx.struct_span_lint(NO_MANGLE_CONST_ITEMS, it.span, |lint| {
@@ -1371,7 +1350,7 @@ impl<'tcx> LateLintPass<'tcx> for UnreachablePub {
         );
     }
 
-    fn check_struct_field(&mut self, cx: &LateContext<'_>, field: &hir::StructField<'_>) {
+    fn check_field_def(&mut self, cx: &LateContext<'_>, field: &hir::FieldDef<'_>) {
         self.perform_lint(cx, "field", field.hir_id, &field.vis, field.span, false);
     }
 
@@ -1827,7 +1806,8 @@ impl<'tcx> LateLintPass<'tcx> for UnnameableTestItems {
             return;
         }
 
-        if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::rustc_test_marker) {
+        let attrs = cx.tcx.hir().attrs(it.hir_id());
+        if let Some(attr) = cx.sess().find_by_name(attrs, sym::rustc_test_marker) {
             cx.struct_span_lint(UNNAMEABLE_TEST_ITEMS, attr.span, |lint| {
                 lint.build("cannot test inner items").emit()
             });
@@ -2305,7 +2285,7 @@ declare_lint! {
 }
 
 declare_lint_pass!(
-    /// Check for used feature gates in `INCOMPLETE_FEATURES` in `librustc_feature/active.rs`.
+    /// Check for used feature gates in `INCOMPLETE_FEATURES` in `rustc_feature/src/active.rs`.
     IncompleteFeatures => [INCOMPLETE_FEATURES]
 );
 
@@ -2986,3 +2966,88 @@ impl<'tcx> LateLintPass<'tcx> for ClashingExternDeclarations {
         }
     }
 }
+
+declare_lint! {
+    /// The `deref_nullptr` lint detects when an null pointer is dereferenced,
+    /// which causes [undefined behavior].
+    ///
+    /// ### Example
+    ///
+    /// ```rust,no_run
+    /// # #![allow(unused)]
+    /// use std::ptr;
+    /// unsafe {
+    ///     let x = &*ptr::null::<i32>();
+    ///     let x = ptr::addr_of!(*ptr::null::<i32>());
+    ///     let x = *(0 as *const i32);
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Dereferencing a null pointer causes [undefined behavior] even as a place expression,
+    /// like `&*(0 as *const i32)` or `addr_of!(*(0 as *const i32))`.
+    ///
+    /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
+    pub DEREF_NULLPTR,
+    Warn,
+    "detects when an null pointer is dereferenced"
+}
+
+declare_lint_pass!(DerefNullPtr => [DEREF_NULLPTR]);
+
+impl<'tcx> LateLintPass<'tcx> for DerefNullPtr {
+    fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &hir::Expr<'_>) {
+        /// test if expression is a null ptr
+        fn is_null_ptr(cx: &LateContext<'_>, expr: &hir::Expr<'_>) -> bool {
+            match &expr.kind {
+                rustc_hir::ExprKind::Cast(ref expr, ref ty) => {
+                    if let rustc_hir::TyKind::Ptr(_) = ty.kind {
+                        return is_zero(expr) || is_null_ptr(cx, expr);
+                    }
+                }
+                // check for call to `core::ptr::null` or `core::ptr::null_mut`
+                rustc_hir::ExprKind::Call(ref path, _) => {
+                    if let rustc_hir::ExprKind::Path(ref qpath) = path.kind {
+                        if let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id() {
+                            return cx.tcx.is_diagnostic_item(sym::ptr_null, def_id)
+                                || cx.tcx.is_diagnostic_item(sym::ptr_null_mut, def_id);
+                        }
+                    }
+                }
+                _ => {}
+            }
+            false
+        }
+
+        /// test if expression is the literal `0`
+        fn is_zero(expr: &hir::Expr<'_>) -> bool {
+            match &expr.kind {
+                rustc_hir::ExprKind::Lit(ref lit) => {
+                    if let LitKind::Int(a, _) = lit.node {
+                        return a == 0;
+                    }
+                }
+                _ => {}
+            }
+            false
+        }
+
+        if let rustc_hir::ExprKind::Unary(ref un_op, ref expr_deref) = expr.kind {
+            if let rustc_hir::UnOp::Deref = un_op {
+                if is_null_ptr(cx, expr_deref) {
+                    cx.struct_span_lint(DEREF_NULLPTR, expr.span, |lint| {
+                        let mut err = lint.build("dereferencing a null pointer");
+                        err.span_label(
+                            expr.span,
+                            "this code causes undefined behavior when executed",
+                        );
+                        err.emit();
+                    });
+                }
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs
index 7d5577cdca6..b3a19bfbf75 100644
--- a/compiler/rustc_lint/src/context.rs
+++ b/compiler/rustc_lint/src/context.rs
@@ -45,6 +45,7 @@ use rustc_target::abi::LayoutOf;
 use tracing::debug;
 
 use std::cell::Cell;
+use std::iter;
 use std::slice;
 
 /// Information about the registered lints.
@@ -100,6 +101,11 @@ enum TargetLint {
     /// Lint with this name existed previously, but has been removed/deprecated.
     /// The string argument is the reason for removal.
     Removed(String),
+
+    /// A lint name that should give no warnings and have no effect.
+    ///
+    /// This is used by rustc to avoid warning about old rustdoc lints before rustdoc registers them as tool lints.
+    Ignored,
 }
 
 pub enum FindLintError {
@@ -171,6 +177,7 @@ impl LintStore {
         self.early_passes.push(Box::new(pass));
     }
 
+    /// Used by clippy.
     pub fn register_pre_expansion_pass(
         &mut self,
         pass: impl Fn() -> EarlyLintPassObject + 'static + sync::Send + sync::Sync,
@@ -266,6 +273,33 @@ impl LintStore {
         }
     }
 
+    /// This lint should be available with either the old or the new name.
+    ///
+    /// Using the old name will not give a warning.
+    /// You must register a lint with the new name before calling this function.
+    #[track_caller]
+    pub fn register_alias(&mut self, old_name: &str, new_name: &str) {
+        let target = match self.by_name.get(new_name) {
+            Some(&Id(lint_id)) => lint_id,
+            _ => bug!("cannot add alias {} for lint {} that does not exist", old_name, new_name),
+        };
+        match self.by_name.insert(old_name.to_string(), Id(target)) {
+            None | Some(Ignored) => {}
+            Some(x) => bug!("duplicate specification of lint {} (was {:?})", old_name, x),
+        }
+    }
+
+    /// This lint should give no warning and have no effect.
+    ///
+    /// This is used by rustc to avoid warning about old rustdoc lints before rustdoc registers them as tool lints.
+    #[track_caller]
+    pub fn register_ignored(&mut self, name: &str) {
+        if self.by_name.insert(name.to_string(), Ignored).is_some() {
+            bug!("duplicate specification of lint {}", name);
+        }
+    }
+
+    /// This lint has been renamed; warn about using the new name and apply the lint.
     #[track_caller]
     pub fn register_renamed(&mut self, old_name: &str, new_name: &str) {
         let target = match self.by_name.get(new_name) {
@@ -284,6 +318,7 @@ impl LintStore {
             Some(&Id(lint_id)) => Ok(vec![lint_id]),
             Some(&Renamed(_, lint_id)) => Ok(vec![lint_id]),
             Some(&Removed(_)) => Err(FindLintError::Removed),
+            Some(&Ignored) => Ok(vec![]),
             None => loop {
                 return match self.lint_groups.get(lint_name) {
                     Some(LintGroup { lint_ids, depr, .. }) => {
@@ -427,6 +462,7 @@ impl LintStore {
                 }
             },
             Some(&Id(ref id)) => CheckLintNameResult::Ok(slice::from_ref(id)),
+            Some(&Ignored) => CheckLintNameResult::Ok(&[]),
         }
     }
 
@@ -670,6 +706,12 @@ pub trait LintContext: Sized {
                         json
                     );
                 }
+                BuiltinLintDiagnostics::ProcMacroBackCompat(note) => {
+                    db.note(&note);
+                }
+                BuiltinLintDiagnostics::OrPatternsBackCompat(span,suggestion) => {
+                    db.span_suggestion(span, "use pat2015 to preserve semantics", suggestion, Applicability::MachineApplicable);
+                }
             }
             // Rewrap `db`, and pass control to the user.
             decorate(LintDiagnosticBuilder::new(db));
@@ -711,7 +753,7 @@ impl<'a> EarlyContext<'a> {
             sess,
             krate,
             lint_store,
-            builder: LintLevelsBuilder::new(sess, warn_about_weird_lints, lint_store),
+            builder: LintLevelsBuilder::new(sess, warn_about_weird_lints, lint_store, &krate.attrs),
             buffered,
         }
     }
@@ -824,10 +866,12 @@ impl<'tcx> LateContext<'tcx> {
     ///     // The given `def_id` is that of an `Option` type
     /// }
     /// ```
+    ///
+    /// Used by clippy, but should be replaced by diagnostic items eventually.
     pub fn match_def_path(&self, def_id: DefId, path: &[Symbol]) -> bool {
         let names = self.get_def_path(def_id);
 
-        names.len() == path.len() && names.into_iter().zip(path.iter()).all(|(a, &b)| a == b)
+        names.len() == path.len() && iter::zip(names, path).all(|(a, &b)| a == b)
     }
 
     /// Gets the absolute path of `def_id` as a vector of `Symbol`.
@@ -868,7 +912,7 @@ impl<'tcx> LateContext<'tcx> {
 
             fn print_dyn_existential(
                 self,
-                _predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+                _predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
             ) -> Result<Self::DynExistential, Self::Error> {
                 Ok(())
             }
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index 0c7b843831a..54fcaef414f 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -109,6 +109,7 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
 
     fn visit_anon_const(&mut self, c: &'a ast::AnonConst) {
         run_early_pass!(self, check_anon_const, c);
+        self.check_id(c.id);
         ast_visit::walk_anon_const(self, c);
     }
 
@@ -163,10 +164,10 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         run_early_pass!(self, check_struct_def_post, s);
     }
 
-    fn visit_struct_field(&mut self, s: &'a ast::StructField) {
+    fn visit_field_def(&mut self, s: &'a ast::FieldDef) {
         self.with_lint_attrs(s.id, &s.attrs, |cx| {
-            run_early_pass!(cx, check_struct_field, s);
-            ast_visit::walk_struct_field(cx, s);
+            run_early_pass!(cx, check_field_def, s);
+            ast_visit::walk_field_def(cx, s);
         })
     }
 
diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs
index 26e536e8f1d..9b1a339572e 100644
--- a/compiler/rustc_lint/src/internal.rs
+++ b/compiler/rustc_lint/src/internal.rs
@@ -283,7 +283,7 @@ fn is_doc_keyword(s: Symbol) -> bool {
 
 impl<'tcx> LateLintPass<'tcx> for ExistingDocKeyword {
     fn check_item(&mut self, cx: &LateContext<'_>, item: &rustc_hir::Item<'_>) {
-        for attr in item.attrs {
+        for attr in cx.tcx.hir().attrs(item.hir_id()) {
             if !attr.has_name(sym::doc) {
                 continue;
             }
diff --git a/compiler/rustc_lint/src/late.rs b/compiler/rustc_lint/src/late.rs
index e019b621aa3..d325b5fe7f8 100644
--- a/compiler/rustc_lint/src/late.rs
+++ b/compiler/rustc_lint/src/late.rs
@@ -16,7 +16,6 @@
 
 use crate::{passes::LateLintPassObject, LateContext, LateLintPass, LintStore};
 use rustc_ast as ast;
-use rustc_ast::walk_list;
 use rustc_data_structures::sync::{join, par_iter, ParallelIterator};
 use rustc_hir as hir;
 use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE};
@@ -53,10 +52,11 @@ impl<'tcx, T: LateLintPass<'tcx>> LateContextAndPass<'tcx, T> {
     /// Merge the lints specified by any lint attributes into the
     /// current lint context, call the provided function, then reset the
     /// lints in effect to their previous state.
-    fn with_lint_attrs<F>(&mut self, id: hir::HirId, attrs: &'tcx [ast::Attribute], f: F)
+    fn with_lint_attrs<F>(&mut self, id: hir::HirId, f: F)
     where
         F: FnOnce(&mut Self),
     {
+        let attrs = self.context.tcx.hir().attrs(id);
         let prev = self.context.last_node_with_lint_attrs;
         self.context.last_node_with_lint_attrs = id;
         self.enter_attrs(attrs);
@@ -125,7 +125,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
     }
 
     fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) {
-        self.with_lint_attrs(param.hir_id, &param.attrs, |cx| {
+        self.with_lint_attrs(param.hir_id, |cx| {
             lint_callback!(cx, check_param, param);
             hir_visit::walk_param(cx, param);
         });
@@ -142,7 +142,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         self.context.generics = it.kind.generics();
         let old_cached_typeck_results = self.context.cached_typeck_results.take();
         let old_enclosing_body = self.context.enclosing_body.take();
-        self.with_lint_attrs(it.hir_id(), &it.attrs, |cx| {
+        self.with_lint_attrs(it.hir_id(), |cx| {
             cx.with_param_env(it.hir_id(), |cx| {
                 lint_callback!(cx, check_item, it);
                 hir_visit::walk_item(cx, it);
@@ -155,7 +155,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
     }
 
     fn visit_foreign_item(&mut self, it: &'tcx hir::ForeignItem<'tcx>) {
-        self.with_lint_attrs(it.hir_id(), &it.attrs, |cx| {
+        self.with_lint_attrs(it.hir_id(), |cx| {
             cx.with_param_env(it.hir_id(), |cx| {
                 lint_callback!(cx, check_foreign_item, it);
                 hir_visit::walk_foreign_item(cx, it);
@@ -170,7 +170,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
     }
 
     fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) {
-        self.with_lint_attrs(e.hir_id, &e.attrs, |cx| {
+        self.with_lint_attrs(e.hir_id, |cx| {
             lint_callback!(cx, check_expr, e);
             hir_visit::walk_expr(cx, e);
             lint_callback!(cx, check_expr_post, e);
@@ -178,11 +178,9 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
     }
 
     fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) {
-        let get_item = |id: hir::ItemId| self.context.tcx.hir().item(id);
-        let attrs = &s.kind.attrs(get_item);
         // See `EarlyContextAndPass::visit_stmt` for an explanation
         // of why we call `walk_stmt` outside of `with_lint_attrs`
-        self.with_lint_attrs(s.hir_id, attrs, |cx| {
+        self.with_lint_attrs(s.hir_id, |cx| {
             lint_callback!(cx, check_stmt, s);
         });
         hir_visit::walk_stmt(self, s);
@@ -221,10 +219,10 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         lint_callback!(self, check_struct_def_post, s);
     }
 
-    fn visit_struct_field(&mut self, s: &'tcx hir::StructField<'tcx>) {
-        self.with_lint_attrs(s.hir_id, &s.attrs, |cx| {
-            lint_callback!(cx, check_struct_field, s);
-            hir_visit::walk_struct_field(cx, s);
+    fn visit_field_def(&mut self, s: &'tcx hir::FieldDef<'tcx>) {
+        self.with_lint_attrs(s.hir_id, |cx| {
+            lint_callback!(cx, check_field_def, s);
+            hir_visit::walk_field_def(cx, s);
         })
     }
 
@@ -234,7 +232,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         g: &'tcx hir::Generics<'tcx>,
         item_id: hir::HirId,
     ) {
-        self.with_lint_attrs(v.id, &v.attrs, |cx| {
+        self.with_lint_attrs(v.id, |cx| {
             lint_callback!(cx, check_variant, v);
             hir_visit::walk_variant(cx, v, g, item_id);
             lint_callback!(cx, check_variant_post, v);
@@ -257,7 +255,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
     }
 
     fn visit_local(&mut self, l: &'tcx hir::Local<'tcx>) {
-        self.with_lint_attrs(l.hir_id, &l.attrs, |cx| {
+        self.with_lint_attrs(l.hir_id, |cx| {
             lint_callback!(cx, check_local, l);
             hir_visit::walk_local(cx, l);
         })
@@ -301,7 +299,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
     fn visit_trait_item(&mut self, trait_item: &'tcx hir::TraitItem<'tcx>) {
         let generics = self.context.generics.take();
         self.context.generics = Some(&trait_item.generics);
-        self.with_lint_attrs(trait_item.hir_id(), &trait_item.attrs, |cx| {
+        self.with_lint_attrs(trait_item.hir_id(), |cx| {
             cx.with_param_env(trait_item.hir_id(), |cx| {
                 lint_callback!(cx, check_trait_item, trait_item);
                 hir_visit::walk_trait_item(cx, trait_item);
@@ -314,7 +312,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
     fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
         let generics = self.context.generics.take();
         self.context.generics = Some(&impl_item.generics);
-        self.with_lint_attrs(impl_item.hir_id(), &impl_item.attrs, |cx| {
+        self.with_lint_attrs(impl_item.hir_id(), |cx| {
             cx.with_param_env(impl_item.hir_id(), |cx| {
                 lint_callback!(cx, check_impl_item, impl_item);
                 hir_visit::walk_impl_item(cx, impl_item);
@@ -334,8 +332,10 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         hir_visit::walk_path(self, p);
     }
 
-    fn visit_attribute(&mut self, attr: &'tcx ast::Attribute) {
-        lint_callback!(self, check_attribute, attr);
+    fn visit_attribute(&mut self, hir_id: hir::HirId, attr: &'tcx ast::Attribute) {
+        self.with_lint_attrs(hir_id, |cx| {
+            lint_callback!(cx, check_attribute, attr);
+        })
     }
 }
 
@@ -396,7 +396,9 @@ fn late_lint_mod_pass<'tcx, T: LateLintPass<'tcx>>(
 
     // Visit the crate attributes
     if hir_id == hir::CRATE_HIR_ID {
-        walk_list!(cx, visit_attribute, tcx.hir().attrs(hir::CRATE_HIR_ID));
+        for attr in tcx.hir().attrs(hir::CRATE_HIR_ID).iter() {
+            cx.visit_attribute(hir_id, attr)
+        }
     }
 }
 
@@ -440,7 +442,7 @@ fn late_lint_pass_crate<'tcx, T: LateLintPass<'tcx>>(tcx: TyCtxt<'tcx>, pass: T)
     let mut cx = LateContextAndPass { context, pass };
 
     // Visit the whole crate.
-    cx.with_lint_attrs(hir::CRATE_HIR_ID, &krate.item.attrs, |cx| {
+    cx.with_lint_attrs(hir::CRATE_HIR_ID, |cx| {
         // since the root module isn't visited as an item (because it isn't an
         // item), warn for it here.
         lint_callback!(cx, check_crate, krate);
diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs
index dbcab72ddf2..54909381a10 100644
--- a/compiler/rustc_lint/src/levels.rs
+++ b/compiler/rustc_lint/src/levels.rs
@@ -1,13 +1,12 @@
 use crate::context::{CheckLintNameResult, LintStore};
 use crate::late::unerased_lint_store;
 use rustc_ast as ast;
-use rustc_ast::attr;
 use rustc_ast::unwrap_or;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder};
 use rustc_hir as hir;
-use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
+use rustc_hir::def_id::{CrateNum, DefId, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::{intravisit, HirId};
 use rustc_middle::hir::map::Map;
 use rustc_middle::lint::LevelAndSource;
@@ -32,13 +31,14 @@ use std::cmp;
 fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> LintLevelMap {
     assert_eq!(cnum, LOCAL_CRATE);
     let store = unerased_lint_store(tcx);
-    let levels = LintLevelsBuilder::new(tcx.sess, false, &store);
+    let crate_attrs = tcx.get_attrs(DefId { krate: cnum, index: CRATE_DEF_INDEX });
+    let levels = LintLevelsBuilder::new(tcx.sess, false, &store, crate_attrs);
     let mut builder = LintLevelMapBuilder { levels, tcx, store };
     let krate = tcx.hir().krate();
 
     builder.levels.id_to_set.reserve(krate.exported_macros.len() + 1);
 
-    let push = builder.levels.push(&krate.item.attrs, &store, true);
+    let push = builder.levels.push(tcx.hir().attrs(hir::CRATE_HIR_ID), &store, true);
     builder.levels.register_id(hir::CRATE_HIR_ID);
     for macro_def in krate.exported_macros {
         builder.levels.register_id(macro_def.hir_id());
@@ -56,6 +56,7 @@ pub struct LintLevelsBuilder<'s> {
     cur: u32,
     warn_about_weird_lints: bool,
     store: &'s LintStore,
+    crate_attrs: &'s [ast::Attribute],
 }
 
 pub struct BuilderPush {
@@ -64,7 +65,12 @@ pub struct BuilderPush {
 }
 
 impl<'s> LintLevelsBuilder<'s> {
-    pub fn new(sess: &'s Session, warn_about_weird_lints: bool, store: &'s LintStore) -> Self {
+    pub fn new(
+        sess: &'s Session,
+        warn_about_weird_lints: bool,
+        store: &'s LintStore,
+        crate_attrs: &'s [ast::Attribute],
+    ) -> Self {
         let mut builder = LintLevelsBuilder {
             sess,
             sets: LintLevelSets::new(),
@@ -72,6 +78,7 @@ impl<'s> LintLevelsBuilder<'s> {
             id_to_set: Default::default(),
             warn_about_weird_lints,
             store,
+            crate_attrs,
         };
         builder.process_command_line(sess, store);
         assert_eq!(builder.sets.list.len(), 1);
@@ -229,10 +236,9 @@ impl<'s> LintLevelsBuilder<'s> {
                 Some(lvl) => lvl,
             };
 
-            let meta = unwrap_or!(attr.meta(), continue);
             self.sess.mark_attr_used(attr);
 
-            let mut metas = unwrap_or!(meta.meta_item_list(), continue);
+            let mut metas = unwrap_or!(attr.meta_item_list(), continue);
 
             if metas.is_empty() {
                 // FIXME (#55112): issue unused-attributes lint for `#[level()]`
@@ -248,8 +254,6 @@ impl<'s> LintLevelsBuilder<'s> {
                     ast::MetaItemKind::Word => {} // actual lint names handled later
                     ast::MetaItemKind::NameValue(ref name_value) => {
                         if item.path == sym::reason {
-                            // found reason, reslice meta list to exclude it
-                            metas = &metas[0..metas.len() - 1];
                             // FIXME (#55112): issue unused-attributes lint if we thereby
                             // don't have any lint names (`#[level(reason = "foo")]`)
                             if let ast::LitKind::Str(rationale, _) = name_value.kind {
@@ -268,6 +272,8 @@ impl<'s> LintLevelsBuilder<'s> {
                                     .span_label(name_value.span, "reason must be a string literal")
                                     .emit();
                             }
+                            // found reason, reslice meta list to exclude it
+                            metas.pop().unwrap();
                         } else {
                             bad_attr(item.span)
                                 .span_label(item.span, "bad attribute argument")
@@ -281,10 +287,10 @@ impl<'s> LintLevelsBuilder<'s> {
             }
 
             for li in metas {
-                let meta_item = match li.meta_item() {
-                    Some(meta_item) if meta_item.is_word() => meta_item,
+                let sp = li.span();
+                let mut meta_item = match li {
+                    ast::NestedMetaItem::MetaItem(meta_item) if meta_item.is_word() => meta_item,
                     _ => {
-                        let sp = li.span();
                         let mut err = bad_attr(sp);
                         let mut add_label = true;
                         if let Some(item) = li.meta_item() {
@@ -304,27 +310,38 @@ impl<'s> LintLevelsBuilder<'s> {
                 };
                 let tool_name = if meta_item.path.segments.len() > 1 {
                     let tool_ident = meta_item.path.segments[0].ident;
-                    if !attr::is_known_lint_tool(tool_ident) {
-                        struct_span_err!(
+                    if !is_known_lint_tool(tool_ident.name, sess, &self.crate_attrs) {
+                        let mut err = struct_span_err!(
                             sess,
                             tool_ident.span,
                             E0710,
-                            "an unknown tool name found in scoped lint: `{}`",
+                            "unknown tool name `{}` found in scoped lint: `{}`",
+                            tool_ident.name,
                             pprust::path_to_string(&meta_item.path),
-                        )
-                        .emit();
+                        );
+                        if sess.is_nightly_build() {
+                            err.help(&format!(
+                                "add `#![register_tool({})]` to the crate root",
+                                tool_ident.name
+                            ));
+                        }
+                        err.emit();
                         continue;
                     }
 
-                    Some(tool_ident.name)
+                    Some(meta_item.path.segments.remove(0).ident.name)
                 } else {
                     None
                 };
-                let name = meta_item.path.segments.last().expect("empty lint name").ident.name;
-                let lint_result = store.check_lint_name(&name.as_str(), tool_name);
+                let name = pprust::path_to_string(&meta_item.path);
+                let lint_result = store.check_lint_name(&name, tool_name);
                 match &lint_result {
                     CheckLintNameResult::Ok(ids) => {
-                        let src = LintLevelSource::Node(name, li.span(), reason);
+                        let src = LintLevelSource::Node(
+                            meta_item.path.segments.last().expect("empty lint name").ident.name,
+                            sp,
+                            reason,
+                        );
                         for &id in *ids {
                             self.check_gated_lint(id, attr.span);
                             self.insert_spec(&mut specs, id, (level, src));
@@ -337,7 +354,7 @@ impl<'s> LintLevelsBuilder<'s> {
                                 let complete_name = &format!("{}::{}", tool_name.unwrap(), name);
                                 let src = LintLevelSource::Node(
                                     Symbol::intern(complete_name),
-                                    li.span(),
+                                    sp,
                                     reason,
                                 );
                                 for id in ids {
@@ -353,7 +370,7 @@ impl<'s> LintLevelsBuilder<'s> {
                                     lint,
                                     lvl,
                                     src,
-                                    Some(li.span().into()),
+                                    Some(sp.into()),
                                     |lint| {
                                         let msg = format!(
                                             "lint name `{}` is deprecated \
@@ -362,7 +379,7 @@ impl<'s> LintLevelsBuilder<'s> {
                                         );
                                         lint.build(&msg)
                                             .span_suggestion(
-                                                li.span(),
+                                                sp,
                                                 "change it to",
                                                 new_lint_name.to_string(),
                                                 Applicability::MachineApplicable,
@@ -373,7 +390,7 @@ impl<'s> LintLevelsBuilder<'s> {
 
                                 let src = LintLevelSource::Node(
                                     Symbol::intern(&new_lint_name),
-                                    li.span(),
+                                    sp,
                                     reason,
                                 );
                                 for id in ids {
@@ -400,12 +417,12 @@ impl<'s> LintLevelsBuilder<'s> {
                             lint,
                             renamed_lint_level,
                             src,
-                            Some(li.span().into()),
+                            Some(sp.into()),
                             |lint| {
                                 let mut err = lint.build(&msg);
                                 if let Some(new_name) = &renamed {
                                     err.span_suggestion(
-                                        li.span(),
+                                        sp,
                                         "use the new name",
                                         new_name.to_string(),
                                         Applicability::MachineApplicable,
@@ -419,30 +436,23 @@ impl<'s> LintLevelsBuilder<'s> {
                         let lint = builtin::UNKNOWN_LINTS;
                         let (level, src) =
                             self.sets.get_lint_level(lint, self.cur, Some(&specs), self.sess);
-                        struct_lint_level(
-                            self.sess,
-                            lint,
-                            level,
-                            src,
-                            Some(li.span().into()),
-                            |lint| {
-                                let name = if let Some(tool_name) = tool_name {
-                                    format!("{}::{}", tool_name, name)
-                                } else {
-                                    name.to_string()
-                                };
-                                let mut db = lint.build(&format!("unknown lint: `{}`", name));
-                                if let Some(suggestion) = suggestion {
-                                    db.span_suggestion(
-                                        li.span(),
-                                        "did you mean",
-                                        suggestion.to_string(),
-                                        Applicability::MachineApplicable,
-                                    );
-                                }
-                                db.emit();
-                            },
-                        );
+                        struct_lint_level(self.sess, lint, level, src, Some(sp.into()), |lint| {
+                            let name = if let Some(tool_name) = tool_name {
+                                format!("{}::{}", tool_name, name)
+                            } else {
+                                name.to_string()
+                            };
+                            let mut db = lint.build(&format!("unknown lint: `{}`", name));
+                            if let Some(suggestion) = suggestion {
+                                db.span_suggestion(
+                                    sp,
+                                    "did you mean",
+                                    suggestion.to_string(),
+                                    Applicability::MachineApplicable,
+                                );
+                            }
+                            db.emit();
+                        });
                     }
                 }
                 // If this lint was renamed, apply the new lint instead of ignoring the attribute.
@@ -450,15 +460,15 @@ impl<'s> LintLevelsBuilder<'s> {
                 // we don't warn about the name change.
                 if let CheckLintNameResult::Warning(_, Some(new_name)) = lint_result {
                     // Ignore any errors or warnings that happen because the new name is inaccurate
-                    if let CheckLintNameResult::Ok(ids) =
-                        store.check_lint_name(&new_name, tool_name)
-                    {
-                        let src =
-                            LintLevelSource::Node(Symbol::intern(&new_name), li.span(), reason);
+                    // NOTE: `new_name` already includes the tool name, so we don't have to add it again.
+                    if let CheckLintNameResult::Ok(ids) = store.check_lint_name(&new_name, None) {
+                        let src = LintLevelSource::Node(Symbol::intern(&new_name), sp, reason);
                         for &id in ids {
                             self.check_gated_lint(id, attr.span);
                             self.insert_spec(&mut specs, id, (level, src));
                         }
+                    } else {
+                        panic!("renamed lint does not exist: {}", new_name);
                     }
                 }
             }
@@ -550,15 +560,25 @@ impl<'s> LintLevelsBuilder<'s> {
         self.id_to_set.insert(id, self.cur);
     }
 
-    pub fn build(self) -> LintLevelSets {
-        self.sets
-    }
-
     pub fn build_map(self) -> LintLevelMap {
         LintLevelMap { sets: self.sets, id_to_set: self.id_to_set }
     }
 }
 
+fn is_known_lint_tool(m_item: Symbol, sess: &Session, attrs: &[ast::Attribute]) -> bool {
+    if [sym::clippy, sym::rustc, sym::rustdoc].contains(&m_item) {
+        return true;
+    }
+    // Look for registered tools
+    // NOTE: does no error handling; error handling is done by rustc_resolve.
+    sess.filter_by_name(attrs, sym::register_tool)
+        .filter_map(|attr| attr.meta_item_list())
+        .flat_map(std::convert::identity)
+        .filter_map(|nested_meta| nested_meta.ident())
+        .map(|ident| ident.name)
+        .any(|name| name == m_item)
+}
+
 struct LintLevelMapBuilder<'a, 'tcx> {
     levels: LintLevelsBuilder<'tcx>,
     tcx: TyCtxt<'tcx>,
@@ -566,11 +586,12 @@ struct LintLevelMapBuilder<'a, 'tcx> {
 }
 
 impl LintLevelMapBuilder<'_, '_> {
-    fn with_lint_attrs<F>(&mut self, id: hir::HirId, attrs: &[ast::Attribute], f: F)
+    fn with_lint_attrs<F>(&mut self, id: hir::HirId, f: F)
     where
         F: FnOnce(&mut Self),
     {
         let is_crate_hir = id == hir::CRATE_HIR_ID;
+        let attrs = self.tcx.hir().attrs(id);
         let push = self.levels.push(attrs, self.store, is_crate_hir);
         if push.changed {
             self.levels.register_id(id);
@@ -588,19 +609,19 @@ impl<'tcx> intravisit::Visitor<'tcx> for LintLevelMapBuilder<'_, 'tcx> {
     }
 
     fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) {
-        self.with_lint_attrs(param.hir_id, &param.attrs, |builder| {
+        self.with_lint_attrs(param.hir_id, |builder| {
             intravisit::walk_param(builder, param);
         });
     }
 
     fn visit_item(&mut self, it: &'tcx hir::Item<'tcx>) {
-        self.with_lint_attrs(it.hir_id(), &it.attrs, |builder| {
+        self.with_lint_attrs(it.hir_id(), |builder| {
             intravisit::walk_item(builder, it);
         });
     }
 
     fn visit_foreign_item(&mut self, it: &'tcx hir::ForeignItem<'tcx>) {
-        self.with_lint_attrs(it.hir_id(), &it.attrs, |builder| {
+        self.with_lint_attrs(it.hir_id(), |builder| {
             intravisit::walk_foreign_item(builder, it);
         })
     }
@@ -613,14 +634,14 @@ impl<'tcx> intravisit::Visitor<'tcx> for LintLevelMapBuilder<'_, 'tcx> {
     }
 
     fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) {
-        self.with_lint_attrs(e.hir_id, &e.attrs, |builder| {
+        self.with_lint_attrs(e.hir_id, |builder| {
             intravisit::walk_expr(builder, e);
         })
     }
 
-    fn visit_struct_field(&mut self, s: &'tcx hir::StructField<'tcx>) {
-        self.with_lint_attrs(s.hir_id, &s.attrs, |builder| {
-            intravisit::walk_struct_field(builder, s);
+    fn visit_field_def(&mut self, s: &'tcx hir::FieldDef<'tcx>) {
+        self.with_lint_attrs(s.hir_id, |builder| {
+            intravisit::walk_field_def(builder, s);
         })
     }
 
@@ -630,31 +651,31 @@ impl<'tcx> intravisit::Visitor<'tcx> for LintLevelMapBuilder<'_, 'tcx> {
         g: &'tcx hir::Generics<'tcx>,
         item_id: hir::HirId,
     ) {
-        self.with_lint_attrs(v.id, &v.attrs, |builder| {
+        self.with_lint_attrs(v.id, |builder| {
             intravisit::walk_variant(builder, v, g, item_id);
         })
     }
 
     fn visit_local(&mut self, l: &'tcx hir::Local<'tcx>) {
-        self.with_lint_attrs(l.hir_id, &l.attrs, |builder| {
+        self.with_lint_attrs(l.hir_id, |builder| {
             intravisit::walk_local(builder, l);
         })
     }
 
     fn visit_arm(&mut self, a: &'tcx hir::Arm<'tcx>) {
-        self.with_lint_attrs(a.hir_id, &a.attrs, |builder| {
+        self.with_lint_attrs(a.hir_id, |builder| {
             intravisit::walk_arm(builder, a);
         })
     }
 
     fn visit_trait_item(&mut self, trait_item: &'tcx hir::TraitItem<'tcx>) {
-        self.with_lint_attrs(trait_item.hir_id(), &trait_item.attrs, |builder| {
+        self.with_lint_attrs(trait_item.hir_id(), |builder| {
             intravisit::walk_trait_item(builder, trait_item);
         });
     }
 
     fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
-        self.with_lint_attrs(impl_item.hir_id(), &impl_item.attrs, |builder| {
+        self.with_lint_attrs(impl_item.hir_id(), |builder| {
             intravisit::walk_impl_item(builder, impl_item);
         });
     }
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index 547779dd685..2f46969b021 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -33,9 +33,10 @@
 #![feature(box_patterns)]
 #![feature(crate_visibility_modifier)]
 #![feature(iter_order_by)]
+#![feature(iter_zip)]
 #![feature(never_type)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(half_open_range_patterns)]
 #![feature(exclusive_range_pattern)]
 #![feature(control_flow_enum)]
@@ -57,6 +58,7 @@ mod methods;
 mod non_ascii_idents;
 mod non_fmt_panic;
 mod nonstandard_style;
+mod noop_method_call;
 mod passes;
 mod redundant_semicolon;
 mod traits;
@@ -81,6 +83,7 @@ use methods::*;
 use non_ascii_idents::*;
 use non_fmt_panic::NonPanicFmt;
 use nonstandard_style::*;
+use noop_method_call::*;
 use redundant_semicolon::*;
 use traits::*;
 use types::*;
@@ -168,6 +171,7 @@ macro_rules! late_lint_passes {
                 DropTraitConstraints: DropTraitConstraints,
                 TemporaryCStringAsPtr: TemporaryCStringAsPtr,
                 NonPanicFmt: NonPanicFmt,
+                NoopMethodCall: NoopMethodCall,
             ]
         );
     };
@@ -202,6 +206,7 @@ macro_rules! late_lint_mod_passes {
                 UnreachablePub: UnreachablePub,
                 ExplicitOutlivesRequirements: ExplicitOutlivesRequirements,
                 InvalidValue: InvalidValue,
+                DerefNullPtr: DerefNullPtr,
             ]
         );
     };
@@ -322,6 +327,7 @@ fn register_builtins(store: &mut LintStore, no_interleave_lints: bool) {
     store.register_renamed("exceeding_bitshifts", "arithmetic_overflow");
     store.register_renamed("redundant_semicolon", "redundant_semicolons");
     store.register_renamed("overlapping_patterns", "overlapping_range_endpoints");
+    store.register_renamed("safe_packed_borrows", "unaligned_references");
 
     // These were moved to tool lints, but rustc still sees them when compiling normally, before
     // tool lints are registered, so `check_tool_name_for_backwards_compat` doesn't work. Use
@@ -337,12 +343,13 @@ fn register_builtins(store: &mut LintStore, no_interleave_lints: bool) {
         "non_autolinks",
     ];
     for rustdoc_lint in RUSTDOC_LINTS {
-        store.register_removed(rustdoc_lint, &format!("use `rustdoc::{}` instead", rustdoc_lint));
+        store.register_ignored(rustdoc_lint);
     }
     store.register_removed(
         "intra_doc_link_resolution_failure",
         "use `rustdoc::broken_intra_doc_links` instead",
     );
+    store.register_removed("rustdoc", "use `rustdoc::all` instead");
 
     store.register_removed("unknown_features", "replaced by an error");
     store.register_removed("unsigned_negation", "replaced by negate_unsigned feature gate");
diff --git a/compiler/rustc_lint/src/non_ascii_idents.rs b/compiler/rustc_lint/src/non_ascii_idents.rs
index a1c7e47e749..301e607fc58 100644
--- a/compiler/rustc_lint/src/non_ascii_idents.rs
+++ b/compiler/rustc_lint/src/non_ascii_idents.rs
@@ -10,7 +10,6 @@ declare_lint! {
     ///
     /// ```rust,compile_fail
     /// # #![allow(unused)]
-    /// #![feature(non_ascii_idents)]
     /// #![deny(non_ascii_idents)]
     /// fn main() {
     ///     let föö = 1;
@@ -21,14 +20,11 @@ declare_lint! {
     ///
     /// ### Explanation
     ///
-    /// Currently on stable Rust, identifiers must contain ASCII characters.
-    /// The [`non_ascii_idents`] nightly-only feature allows identifiers to
-    /// contain non-ASCII characters. This lint allows projects that wish to
-    /// retain the limit of only using ASCII characters to switch this lint to
-    /// "forbid" (for example to ease collaboration or for security reasons).
+    /// This lint allows projects that wish to retain the limit of only using
+    /// ASCII characters to switch this lint to "forbid" (for example to ease
+    /// collaboration or for security reasons).
     /// See [RFC 2457] for more details.
     ///
-    /// [`non_ascii_idents`]: https://doc.rust-lang.org/nightly/unstable-book/language-features/non-ascii-idents.html
     /// [RFC 2457]: https://github.com/rust-lang/rfcs/blob/master/text/2457-non-ascii-idents.md
     pub NON_ASCII_IDENTS,
     Allow,
@@ -44,7 +40,6 @@ declare_lint! {
     ///
     /// ```rust
     /// # #![allow(unused)]
-    /// #![feature(non_ascii_idents)]
     /// const µ: f64 = 0.000001;
     /// ```
     ///
@@ -52,10 +47,8 @@ declare_lint! {
     ///
     /// ### Explanation
     ///
-    /// With the [`non_ascii_idents`] nightly-only feature enabled,
-    /// identifiers are allowed to use non-ASCII characters. This lint warns
-    /// about using characters which are not commonly used, and may cause
-    /// visual confusion.
+    /// This lint warns about using characters which are not commonly used, and may
+    /// cause visual confusion.
     ///
     /// This lint is triggered by identifiers that contain a codepoint that is
     /// not part of the set of "Allowed" codepoints as described by [Unicode®
@@ -66,7 +59,6 @@ declare_lint! {
     /// that if you "forbid" this lint that existing code may fail in the
     /// future.
     ///
-    /// [`non_ascii_idents`]: https://doc.rust-lang.org/nightly/unstable-book/language-features/non-ascii-idents.html
     /// [TR39Allowed]: https://www.unicode.org/reports/tr39/#General_Security_Profile
     pub UNCOMMON_CODEPOINTS,
     Warn,
@@ -81,8 +73,6 @@ declare_lint! {
     /// ### Example
     ///
     /// ```rust
-    /// #![feature(non_ascii_idents)]
-    ///
     /// // Latin Capital Letter E With Caron
     /// pub const Ě: i32 = 1;
     /// // Latin Capital Letter E With Breve
@@ -93,10 +83,8 @@ declare_lint! {
     ///
     /// ### Explanation
     ///
-    /// With the [`non_ascii_idents`] nightly-only feature enabled,
-    /// identifiers are allowed to use non-ASCII characters. This lint warns
-    /// when different identifiers may appear visually similar, which can
-    /// cause confusion.
+    /// This lint warns when different identifiers may appear visually similar,
+    /// which can cause confusion.
     ///
     /// The confusable detection algorithm is based on [Unicode® Technical
     /// Standard #39 Unicode Security Mechanisms Section 4 Confusable
@@ -110,7 +98,6 @@ declare_lint! {
     /// Beware that if you "forbid" this lint that existing code may fail in
     /// the future.
     ///
-    /// [`non_ascii_idents`]: https://doc.rust-lang.org/nightly/unstable-book/language-features/non-ascii-idents.html
     /// [TR39Confusable]: https://www.unicode.org/reports/tr39/#Confusable_Detection
     pub CONFUSABLE_IDENTS,
     Warn,
@@ -127,8 +114,6 @@ declare_lint! {
     /// ### Example
     ///
     /// ```rust
-    /// #![feature(non_ascii_idents)]
-    ///
     /// // The Japanese katakana character エ can be confused with the Han character 工.
     /// const エ: &'static str = "アイウ";
     /// ```
@@ -137,10 +122,8 @@ declare_lint! {
     ///
     /// ### Explanation
     ///
-    /// With the [`non_ascii_idents`] nightly-only feature enabled,
-    /// identifiers are allowed to use non-ASCII characters. This lint warns
-    /// when characters between different scripts may appear visually similar,
-    /// which can cause confusion.
+    /// This lint warns when characters between different scripts may appear
+    /// visually similar, which can cause confusion.
     ///
     /// If the crate contains other identifiers in the same script that have
     /// non-confusable characters, then this lint will *not* be issued. For
@@ -152,8 +135,6 @@ declare_lint! {
     /// Note that the set of confusable characters may change over time.
     /// Beware that if you "forbid" this lint that existing code may fail in
     /// the future.
-    ///
-    /// [`non_ascii_idents`]: https://doc.rust-lang.org/nightly/unstable-book/language-features/non-ascii-idents.html
     pub MIXED_SCRIPT_CONFUSABLES,
     Warn,
     "detects Unicode scripts whose mixed script confusables codepoints are solely used",
diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs
index 121dde325f7..be9c6eafb6f 100644
--- a/compiler/rustc_lint/src/nonstandard_style.rs
+++ b/compiler/rustc_lint/src/nonstandard_style.rs
@@ -400,14 +400,15 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase {
                 }
                 _ => (),
             },
-            FnKind::ItemFn(ident, _, header, _, attrs) => {
+            FnKind::ItemFn(ident, _, header, _) => {
+                let attrs = cx.tcx.hir().attrs(id);
                 // Skip foreign-ABI #[no_mangle] functions (Issue #31924)
                 if header.abi != Abi::Rust && cx.sess().contains_name(attrs, sym::no_mangle) {
                     return;
                 }
                 self.check_snake_case(cx, "function", ident);
             }
-            FnKind::Closure(_) => (),
+            FnKind::Closure => (),
         }
     }
 
@@ -504,8 +505,9 @@ impl NonUpperCaseGlobals {
 
 impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
     fn check_item(&mut self, cx: &LateContext<'_>, it: &hir::Item<'_>) {
+        let attrs = cx.tcx.hir().attrs(it.hir_id());
         match it.kind {
-            hir::ItemKind::Static(..) if !cx.sess().contains_name(&it.attrs, sym::no_mangle) => {
+            hir::ItemKind::Static(..) if !cx.sess().contains_name(attrs, sym::no_mangle) => {
                 NonUpperCaseGlobals::check_upper_case(cx, "static variable", &it.ident);
             }
             hir::ItemKind::Const(..) => {
diff --git a/compiler/rustc_lint/src/noop_method_call.rs b/compiler/rustc_lint/src/noop_method_call.rs
new file mode 100644
index 00000000000..479cc00199f
--- /dev/null
+++ b/compiler/rustc_lint/src/noop_method_call.rs
@@ -0,0 +1,111 @@
+use crate::context::LintContext;
+use crate::rustc_middle::ty::TypeFoldable;
+use crate::LateContext;
+use crate::LateLintPass;
+use rustc_hir::def::DefKind;
+use rustc_hir::{Expr, ExprKind};
+use rustc_middle::ty;
+use rustc_span::symbol::sym;
+
+declare_lint! {
+    /// The `noop_method_call` lint detects specific calls to noop methods
+    /// such as a calling `<&T as Clone>::clone` where `T: !Clone`.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// # #![allow(unused)]
+    /// #![warn(noop_method_call)]
+    /// struct Foo;
+    /// let foo = &Foo;
+    /// let clone: &Foo = foo.clone();
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Some method calls are noops meaning that they do nothing. Usually such methods
+    /// are the result of blanket implementations that happen to create some method invocations
+    /// that end up not doing anything. For instance, `Clone` is implemented on all `&T`, but
+    /// calling `clone` on a `&T` where `T` does not implement clone, actually doesn't do anything
+    /// as references are copy. This lint detects these calls and warns the user about them.
+    pub NOOP_METHOD_CALL,
+    Allow,
+    "detects the use of well-known noop methods"
+}
+
+declare_lint_pass!(NoopMethodCall => [NOOP_METHOD_CALL]);
+
+impl<'tcx> LateLintPass<'tcx> for NoopMethodCall {
+    fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>) {
+        // We only care about method calls.
+        let (call, elements) = match expr.kind {
+            ExprKind::MethodCall(call, _, elements, _) => (call, elements),
+            _ => return,
+        };
+        // We only care about method calls corresponding to the `Clone`, `Deref` and `Borrow`
+        // traits and ignore any other method call.
+        let (trait_id, did) = match cx.typeck_results().type_dependent_def(expr.hir_id) {
+            // Verify we are dealing with a method/associated function.
+            Some((DefKind::AssocFn, did)) => match cx.tcx.trait_of_item(did) {
+                // Check that we're dealing with a trait method for one of the traits we care about.
+                Some(trait_id)
+                    if [sym::Clone, sym::Deref, sym::Borrow]
+                        .iter()
+                        .any(|s| cx.tcx.is_diagnostic_item(*s, trait_id)) =>
+                {
+                    (trait_id, did)
+                }
+                _ => return,
+            },
+            _ => return,
+        };
+        let substs = cx.typeck_results().node_substs(expr.hir_id);
+        if substs.needs_subst() {
+            // We can't resolve on types that require monomorphization, so we don't handle them if
+            // we need to perfom substitution.
+            return;
+        }
+        let param_env = cx.tcx.param_env(trait_id);
+        // Resolve the trait method instance.
+        let i = match ty::Instance::resolve(cx.tcx, param_env, did, substs) {
+            Ok(Some(i)) => i,
+            _ => return,
+        };
+        // (Re)check that it implements the noop diagnostic.
+        for s in [sym::noop_method_clone, sym::noop_method_deref, sym::noop_method_borrow].iter() {
+            if cx.tcx.is_diagnostic_item(*s, i.def_id()) {
+                let method = &call.ident.name;
+                let receiver = &elements[0];
+                let receiver_ty = cx.typeck_results().expr_ty(receiver);
+                let expr_ty = cx.typeck_results().expr_ty_adjusted(expr);
+                if receiver_ty != expr_ty {
+                    // This lint will only trigger if the receiver type and resulting expression \
+                    // type are the same, implying that the method call is unnecessary.
+                    return;
+                }
+                let expr_span = expr.span;
+                let note = format!(
+                    "the type `{:?}` which `{}` is being called on is the same as \
+                     the type returned from `{}`, so the method call does not do \
+                     anything and can be removed",
+                    receiver_ty, method, method,
+                );
+
+                let span = expr_span.with_lo(receiver.span.hi());
+                cx.struct_span_lint(NOOP_METHOD_CALL, span, |lint| {
+                    let method = &call.ident.name;
+                    let message = format!(
+                        "call to `.{}()` on a reference in this situation does nothing",
+                        &method,
+                    );
+                    lint.build(&message)
+                        .span_label(span, "unnecessary method call")
+                        .note(&note)
+                        .emit()
+                });
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_lint/src/passes.rs b/compiler/rustc_lint/src/passes.rs
index ffbed3a0aff..bbe17dcf4b7 100644
--- a/compiler/rustc_lint/src/passes.rs
+++ b/compiler/rustc_lint/src/passes.rs
@@ -57,7 +57,7 @@ macro_rules! late_lint_methods {
             fn check_impl_item_post(a: &$hir hir::ImplItem<$hir>);
             fn check_struct_def(a: &$hir hir::VariantData<$hir>);
             fn check_struct_def_post(a: &$hir hir::VariantData<$hir>);
-            fn check_struct_field(a: &$hir hir::StructField<$hir>);
+            fn check_field_def(a: &$hir hir::FieldDef<$hir>);
             fn check_variant(a: &$hir hir::Variant<$hir>);
             fn check_variant_post(a: &$hir hir::Variant<$hir>);
             fn check_lifetime(a: &$hir hir::Lifetime);
@@ -193,7 +193,7 @@ macro_rules! early_lint_methods {
             fn check_impl_item_post(a: &ast::AssocItem);
             fn check_struct_def(a: &ast::VariantData);
             fn check_struct_def_post(a: &ast::VariantData);
-            fn check_struct_field(a: &ast::StructField);
+            fn check_field_def(a: &ast::FieldDef);
             fn check_variant(a: &ast::Variant);
             fn check_variant_post(a: &ast::Variant);
             fn check_lifetime(a: &ast::Lifetime);
diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs
index 792655ff35a..9c94bab04e9 100644
--- a/compiler/rustc_lint/src/types.rs
+++ b/compiler/rustc_lint/src/types.rs
@@ -17,6 +17,7 @@ use rustc_target::abi::{Integer, LayoutOf, TagEncoding, VariantIdx, Variants};
 use rustc_target::spec::abi::Abi as SpecAbi;
 
 use std::cmp;
+use std::iter;
 use std::ops::ControlFlow;
 use tracing::debug;
 
@@ -217,7 +218,11 @@ fn report_bin_hex_error(
     cx.struct_span_lint(OVERFLOWING_LITERALS, expr.span, |lint| {
         let (t, actually) = match ty {
             attr::IntType::SignedInt(t) => {
-                let actually = size.sign_extend(val) as i128;
+                let actually = if negative {
+                    -(size.sign_extend(val) as i128)
+                } else {
+                    size.sign_extend(val) as i128
+                };
                 (t.name_str(), actually.to_string())
             }
             attr::IntType::UnsignedInt(t) => {
@@ -226,11 +231,22 @@ fn report_bin_hex_error(
             }
         };
         let mut err = lint.build(&format!("literal out of range for `{}`", t));
-        err.note(&format!(
-            "the literal `{}` (decimal `{}`) does not fit into \
-             the type `{}` and will become `{}{}`",
-            repr_str, val, t, actually, t
-        ));
+        if negative {
+            // If the value is negative,
+            // emits a note about the value itself, apart from the literal.
+            err.note(&format!(
+                "the literal `{}` (decimal `{}`) does not fit into \
+                 the type `{}`",
+                repr_str, val, t
+            ));
+            err.note(&format!("and the value `-{}` will become `{}{}`", repr_str, actually, t));
+        } else {
+            err.note(&format!(
+                "the literal `{}` (decimal `{}`) does not fit into \
+                 the type `{}` and will become `{}{}`",
+                repr_str, val, t, actually, t
+            ));
+        }
         if let Some(sugg_ty) =
             get_type_suggestion(&cx.typeck_results().node_type(expr.hir_id), val, negative)
         {
@@ -1240,7 +1256,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         let sig = self.cx.tcx.fn_sig(def_id);
         let sig = self.cx.tcx.erase_late_bound_regions(sig);
 
-        for (input_ty, input_hir) in sig.inputs().iter().zip(decl.inputs) {
+        for (input_ty, input_hir) in iter::zip(sig.inputs(), decl.inputs) {
             self.check_type_for_ffi_and_report_errors(input_hir.span, input_ty, false, false);
         }
 
@@ -1340,10 +1356,7 @@ impl<'tcx> LateLintPass<'tcx> for VariantSizeDifferences {
                 layout
             );
 
-            let (largest, slargest, largest_index) = enum_definition
-                .variants
-                .iter()
-                .zip(variants)
+            let (largest, slargest, largest_index) = iter::zip(enum_definition.variants, variants)
                 .map(|(variant, variant_layout)| {
                     // Subtract the size of the enum tag.
                     let bytes = variant_layout.size.bytes().saturating_sub(tag_size);
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index b611aebad01..67946dfb292 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -406,6 +406,8 @@ impl<'tcx> LateLintPass<'tcx> for UnusedAttributes {
         if !cx.sess().is_attr_used(attr) {
             debug!("emitting warning for: {:?}", attr);
             cx.struct_span_lint(UNUSED_ATTRIBUTES, attr.span, |lint| {
+                // Mark as used to avoid duplicate warnings.
+                cx.sess().mark_attr_used(attr);
                 lint.build("unused attribute").emit()
             });
             // Is it a builtin attribute that must be used at the crate level?
@@ -602,7 +604,7 @@ trait UnusedDelimLint {
         use rustc_ast::ExprKind::*;
         let (value, ctx, followed_by_block, left_pos, right_pos) = match e.kind {
             // Do not lint `unused_braces` in `if let` expressions.
-            If(ref cond, ref block, ..)
+            If(ref cond, ref block, _)
                 if !matches!(cond.kind, Let(_, _)) || Self::LINT_EXPR_IN_PATTERN_MATCHING_CTX =>
             {
                 let left = e.span.lo() + rustc_span::BytePos(2);
@@ -816,8 +818,33 @@ impl UnusedParens {
 
 impl EarlyLintPass for UnusedParens {
     fn check_expr(&mut self, cx: &EarlyContext<'_>, e: &ast::Expr) {
-        if let ExprKind::Let(ref pat, ..) | ExprKind::ForLoop(ref pat, ..) = e.kind {
-            self.check_unused_parens_pat(cx, pat, false, false);
+        match e.kind {
+            ExprKind::Let(ref pat, _) | ExprKind::ForLoop(ref pat, ..) => {
+                self.check_unused_parens_pat(cx, pat, false, false);
+            }
+            // We ignore parens in cases like `if (((let Some(0) = Some(1))))` because we already
+            // handle a hard error for them during AST lowering in `lower_expr_mut`, but we still
+            // want to complain about things like `if let 42 = (42)`.
+            ExprKind::If(ref cond, ref block, ref else_)
+                if matches!(cond.peel_parens().kind, ExprKind::Let(..)) =>
+            {
+                self.check_unused_delims_expr(
+                    cx,
+                    cond.peel_parens(),
+                    UnusedDelimsCtx::LetScrutineeExpr,
+                    true,
+                    None,
+                    None,
+                );
+                for stmt in &block.stmts {
+                    <Self as UnusedDelimLint>::check_stmt(self, cx, stmt);
+                }
+                if let Some(e) = else_ {
+                    <Self as UnusedDelimLint>::check_expr(self, cx, e);
+                }
+                return;
+            }
+            _ => {}
         }
 
         <Self as UnusedDelimLint>::check_expr(self, cx, e)
@@ -847,7 +874,7 @@ impl EarlyLintPass for UnusedParens {
 
     fn check_stmt(&mut self, cx: &EarlyContext<'_>, s: &ast::Stmt) {
         if let StmtKind::Local(ref local) = s.kind {
-            self.check_unused_parens_pat(cx, &local.pat, false, false);
+            self.check_unused_parens_pat(cx, &local.pat, true, false);
         }
 
         <Self as UnusedDelimLint>::check_stmt(self, cx, s)
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index 12d849e3b94..f15a7cc5ec2 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -6,9 +6,8 @@
 //! compiler code, rather than using their own custom pass. Those
 //! lints are all available in `rustc_lint::builtin`.
 
-use crate::{declare_lint, declare_lint_pass};
+use crate::{declare_lint, declare_lint_pass, FutureBreakage};
 use rustc_span::edition::Edition;
-use rustc_span::symbol::sym;
 
 declare_lint! {
     /// The `forbidden_lint_groups` lint detects violations of
@@ -548,7 +547,7 @@ declare_lint! {
     /// Also consider if you intended to use an _inner attribute_ (with a `!`
     /// such as `#![allow(unused)]`) which applies to the item the attribute
     /// is within, or an _outer attribute_ (without a `!` such as
-    /// `#[allow(unsued)]`) which applies to the item *following* the
+    /// `#[allow(unused)]`) which applies to the item *following* the
     /// attribute.
     ///
     /// [attributes]: https://doc.rust-lang.org/reference/attributes.html
@@ -1058,6 +1057,7 @@ declare_lint! {
     ///     unsafe {
     ///         let foo = Foo { field1: 0, field2: 0 };
     ///         let _ = &foo.field1;
+    ///         println!("{}", foo.field1); // An implicit `&` is added here, triggering the lint.
     ///     }
     /// }
     /// ```
@@ -1066,20 +1066,20 @@ declare_lint! {
     ///
     /// ### Explanation
     ///
-    /// Creating a reference to an insufficiently aligned packed field is
-    /// [undefined behavior] and should be disallowed.
-    ///
-    /// This lint is "allow" by default because there is no stable
-    /// alternative, and it is not yet certain how widespread existing code
-    /// will trigger this lint.
-    ///
-    /// See [issue #27060] for more discussion.
+    /// Creating a reference to an insufficiently aligned packed field is [undefined behavior] and
+    /// should be disallowed. Using an `unsafe` block does not change anything about this. Instead,
+    /// the code should do a copy of the data in the packed field or use raw pointers and unaligned
+    /// accesses. See [issue #82523] for more information.
     ///
     /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
-    /// [issue #27060]: https://github.com/rust-lang/rust/issues/27060
+    /// [issue #82523]: https://github.com/rust-lang/rust/issues/82523
     pub UNALIGNED_REFERENCES,
-    Allow,
+    Warn,
     "detects unaligned references to fields of packed structs",
+    @future_incompatible = FutureIncompatibleInfo {
+        reference: "issue #82523 <https://github.com/rust-lang/rust/issues/82523>",
+        edition: None,
+    };
     report_in_external_macro
 }
 
@@ -1152,49 +1152,6 @@ declare_lint! {
 }
 
 declare_lint! {
-    /// The `safe_packed_borrows` lint detects borrowing a field in the
-    /// interior of a packed structure with alignment other than 1.
-    ///
-    /// ### Example
-    ///
-    /// ```rust
-    /// #[repr(packed)]
-    /// pub struct Unaligned<T>(pub T);
-    ///
-    /// pub struct Foo {
-    ///     start: u8,
-    ///     data: Unaligned<u32>,
-    /// }
-    ///
-    /// fn main() {
-    ///     let x = Foo { start: 0, data: Unaligned(1) };
-    ///     let y = &x.data.0;
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    /// ### Explanation
-    ///
-    /// This type of borrow is unsafe and can cause errors on some platforms
-    /// and violates some assumptions made by the compiler. This was
-    /// previously allowed unintentionally. This is a [future-incompatible]
-    /// lint to transition this to a hard error in the future. See [issue
-    /// #46043] for more details, including guidance on how to solve the
-    /// problem.
-    ///
-    /// [issue #46043]: https://github.com/rust-lang/rust/issues/46043
-    /// [future-incompatible]: ../index.md#future-incompatible-lints
-    pub SAFE_PACKED_BORROWS,
-    Warn,
-    "safe borrows of fields of packed structs were erroneously allowed",
-    @future_incompatible = FutureIncompatibleInfo {
-        reference: "issue #46043 <https://github.com/rust-lang/rust/issues/46043>",
-        edition: None,
-    };
-}
-
-declare_lint! {
     /// The `patterns_in_fns_without_body` lint detects `mut` identifier
     /// patterns as a parameter in functions without a body.
     ///
@@ -1978,7 +1935,7 @@ declare_lint! {
     Warn,
     "detects proc macro derives using inaccessible names from parent modules",
     @future_incompatible = FutureIncompatibleInfo {
-        reference: "issue #50504 <https://github.com/rust-lang/rust/issues/50504>",
+        reference: "issue #83583 <https://github.com/rust-lang/rust/issues/83583>",
         edition: None,
     };
 }
@@ -2488,17 +2445,58 @@ declare_lint! {
 }
 
 declare_lint! {
-    /// The `unsafe_op_in_unsafe_fn` lint detects unsafe operations in unsafe
-    /// functions without an explicit unsafe block. This lint only works on
-    /// the [**nightly channel**] with the
-    /// `#![feature(unsafe_block_in_unsafe_fn)]` feature.
+    /// The `bad_asm_style` lint detects the use of the `.intel_syntax` and
+    /// `.att_syntax` directives.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,ignore (fails on system llvm)
+    /// #![feature(asm)]
+    ///
+    /// fn main() {
+    ///     #[cfg(target_arch="x86_64")]
+    ///     unsafe {
+    ///         asm!(
+    ///             ".att_syntax",
+    ///             "movl {0}, {0}", in(reg) 0usize
+    ///         );
+    ///     }
+    /// }
+    /// ```
+    ///
+    /// This will produce:
+    ///
+    /// ```text
+    ///  warning: avoid using `.att_syntax`, prefer using `options(att_syntax)` instead
+    ///  --> test.rs:7:14
+    ///   |
+    /// 7 |             ".att_syntax",
+    ///   |              ^^^^^^^^^^^
+    /// 8 |             "movq {0}, {0}", out(reg) _,
+    /// 9 |         );
+    ///   |         - help: add option: `, options(att_syntax)`
+    ///   |
+    ///   = note: `#[warn(bad_asm_style)]` on by default
+    /// ```
+    ///
+    /// ### Explanation
     ///
-    /// [**nightly channel**]: https://doc.rust-lang.org/book/appendix-07-nightly-rust.html
+    /// On x86, `asm!` uses the intel assembly syntax by default. While this
+    /// can be switched using assembler directives like `.att_syntax`, using the
+    /// `att_syntax` option is recommended instead because it will also properly
+    /// prefix register placeholders with `%` as required by AT&T syntax.
+    pub BAD_ASM_STYLE,
+    Warn,
+    "incorrect use of inline assembly",
+}
+
+declare_lint! {
+    /// The `unsafe_op_in_unsafe_fn` lint detects unsafe operations in unsafe
+    /// functions without an explicit unsafe block.
     ///
     /// ### Example
     ///
     /// ```rust,compile_fail
-    /// #![feature(unsafe_block_in_unsafe_fn)]
     /// #![deny(unsafe_op_in_unsafe_fn)]
     ///
     /// unsafe fn foo() {}
@@ -2524,9 +2522,10 @@ declare_lint! {
     ///
     /// The fix to this is to wrap the unsafe code in an `unsafe` block.
     ///
-    /// This lint is "allow" by default because it has not yet been
-    /// stabilized, and is not yet complete. See [RFC #2585] and [issue
-    /// #71668] for more details
+    /// This lint is "allow" by default since this will affect a large amount
+    /// of existing code, and the exact plan for increasing the severity is
+    /// still being considered. See [RFC #2585] and [issue #71668] for more
+    /// details.
     ///
     /// [`unsafe fn`]: https://doc.rust-lang.org/reference/unsafe-functions.html
     /// [`unsafe` block]: https://doc.rust-lang.org/reference/expressions/block-expr.html#unsafe-blocks
@@ -2536,7 +2535,6 @@ declare_lint! {
     pub UNSAFE_OP_IN_UNSAFE_FN,
     Allow,
     "unsafe operations in unsafe functions without an explicit unsafe block are deprecated",
-    @feature_gate = sym::unsafe_block_in_unsafe_fn;
 }
 
 declare_lint! {
@@ -2680,7 +2678,7 @@ declare_lint! {
     /// Statics with an uninhabited type can never be initialized, so they are impossible to define.
     /// However, this can be side-stepped with an `extern static`, leading to problems later in the
     /// compiler which assumes that there are no initialized uninhabited places (such as locals or
-    /// statics). This was accientally allowed, but is being phased out.
+    /// statics). This was accidentally allowed, but is being phased out.
     pub UNINHABITED_STATIC,
     Warn,
     "uninhabited static",
@@ -2914,7 +2912,6 @@ declare_lint_pass! {
         RENAMED_AND_REMOVED_LINTS,
         UNALIGNED_REFERENCES,
         CONST_ITEM_MUTATION,
-        SAFE_PACKED_BORROWS,
         PATTERNS_IN_FNS_WITHOUT_BODY,
         MISSING_FRAGMENT_SPECIFIER,
         LATE_BOUND_LIFETIME_ARGUMENTS,
@@ -2948,6 +2945,7 @@ declare_lint_pass! {
         NONTRIVIAL_STRUCTURAL_MATCH,
         SOFT_UNSTABLE,
         INLINE_NO_SANITIZE,
+        BAD_ASM_STYLE,
         ASM_SUB_REGISTER,
         UNSAFE_OP_IN_UNSAFE_FN,
         INCOMPLETE_INCLUDE,
@@ -2962,6 +2960,8 @@ declare_lint_pass! {
         SEMICOLON_IN_EXPRESSIONS_FROM_MACROS,
         DISJOINT_CAPTURE_DROP_REORDER,
         LEGACY_DERIVE_HELPERS,
+        PROC_MACRO_BACK_COMPAT,
+        OR_PATTERNS_BACK_COMPAT,
     ]
 }
 
@@ -3059,3 +3059,117 @@ declare_lint! {
     Allow,
     "No declared ABI for extern declaration"
 }
+
+declare_lint! {
+    /// The `invalid_doc_attributes` lint detects when the `#[doc(...)]` is
+    /// misused.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(warnings)]
+    ///
+    /// pub mod submodule {
+    ///     #![doc(test(no_crate_inject))]
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Previously, there were very like checks being performed on `#[doc(..)]`
+    /// unlike the other attributes. It'll now catch all the issues that it
+    /// silently ignored previously.
+    pub INVALID_DOC_ATTRIBUTES,
+    Warn,
+    "detects invalid `#[doc(...)]` attributes",
+    @future_incompatible = FutureIncompatibleInfo {
+        reference: "issue #82730 <https://github.com/rust-lang/rust/issues/82730>",
+        edition: None,
+    };
+}
+
+declare_lint! {
+    /// The `proc_macro_back_compat` lint detects uses of old versions of certain
+    /// proc-macro crates, which have hardcoded workarounds in the compiler.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,ignore (needs-dependency)
+    ///
+    /// use time_macros_impl::impl_macros;
+    /// struct Foo;
+    /// impl_macros!(Foo);
+    /// ```
+    ///
+    /// This will produce:
+    ///
+    /// ```text
+    /// warning: using an old version of `time-macros-impl`
+    ///   ::: $DIR/group-compat-hack.rs:27:5
+    ///    |
+    /// LL |     impl_macros!(Foo);
+    ///    |     ------------------ in this macro invocation
+    ///    |
+    ///    = note: `#[warn(proc_macro_back_compat)]` on by default
+    ///    = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+    ///    = note: for more information, see issue #83125 <https://github.com/rust-lang/rust/issues/83125>
+    ///    = note: the `time-macros-impl` crate will stop compiling in futures version of Rust. Please update to the latest version of the `time` crate to avoid breakage
+    ///    = note: this warning originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
+    /// ```
+    ///
+    /// ### Explanation
+    ///
+    /// Eventually, the backwards-compatibility hacks present in the compiler will be removed,
+    /// causing older versions of certain crates to stop compiling.
+    /// This is a [future-incompatible] lint to ease the transition to an error.
+    /// See [issue #83125] for more details.
+    ///
+    /// [issue #83125]: https://github.com/rust-lang/rust/issues/83125
+    /// [future-incompatible]: ../index.md#future-incompatible-lints
+    pub PROC_MACRO_BACK_COMPAT,
+    Warn,
+    "detects usage of old versions of certain proc-macro crates",
+    @future_incompatible = FutureIncompatibleInfo {
+        reference: "issue #83125 <https://github.com/rust-lang/rust/issues/83125>",
+        edition: None,
+        future_breakage: Some(FutureBreakage {
+            date: None
+        })
+    };
+}
+
+declare_lint! {
+    /// The `or_patterns_back_compat` lint detects usage of old versions of or-patterns.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(or_patterns_back_compat)]
+    /// macro_rules! match_any {
+    ///     ( $expr:expr , $( $( $pat:pat )|+ => $expr_arm:expr ),+ ) => {
+    ///         match $expr {
+    ///             $(
+    ///                 $( $pat => $expr_arm, )+
+    ///             )+
+    ///         }
+    ///     };
+    /// }
+    ///
+    /// fn main() {
+    ///     let result: Result<i64, i32> = Err(42);
+    ///     let int: i64 = match_any!(result, Ok(i) | Err(i) => i.into());
+    ///     assert_eq!(int, 42);
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// In Rust 2021, the pat matcher will match new patterns, which include the | character.
+    pub OR_PATTERNS_BACK_COMPAT,
+    Allow,
+    "detects usage of old versions of or-patterns",
+}
diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs
index 4c7d3f6c8c0..70475563a4a 100644
--- a/compiler/rustc_lint_defs/src/lib.rs
+++ b/compiler/rustc_lint_defs/src/lib.rs
@@ -266,6 +266,8 @@ pub enum BuiltinLintDiagnostics {
     PatternsInFnsWithoutBody(Span, Ident),
     LegacyDeriveHelpers(Span),
     ExternDepSpec(String, ExternDepSpec),
+    ProcMacroBackCompat(String),
+    OrPatternsBackCompat(Span, String),
 }
 
 /// Lints that are buffered up early on in the `Session` before the
diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index 621363bed80..301ed639f3b 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -88,16 +88,6 @@ fn main() {
         "riscv",
     ];
 
-    let mut version_cmd = Command::new(&llvm_config);
-    version_cmd.arg("--version");
-    let version_output = output(&mut version_cmd);
-    let mut parts = version_output.split('.').take(2).filter_map(|s| s.parse::<u32>().ok());
-    let (major, _minor) = if let (Some(major), Some(minor)) = (parts.next(), parts.next()) {
-        (major, minor)
-    } else {
-        (8, 0)
-    };
-
     let required_components = &[
         "ipo",
         "bitreader",
@@ -123,10 +113,6 @@ fn main() {
         println!("cargo:rustc-cfg=llvm_component=\"{}\"", component);
     }
 
-    if major >= 9 {
-        println!("cargo:rustc-cfg=llvm_has_msp430_asm_parser");
-    }
-
     // Link in our own LLVM shims, compiled with the same flags as LLVM
     let mut cmd = Command::new(&llvm_config);
     cmd.arg("--cxxflags");
diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp
index 2797fe8df4a..97541e615da 100644
--- a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp
@@ -91,11 +91,7 @@ extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) {
 extern "C" LLVMRustArchiveIteratorRef
 LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) {
   Archive *Archive = RustArchive->getBinary();
-#if LLVM_VERSION_GE(10, 0)
   std::unique_ptr<Error> Err = std::make_unique<Error>(Error::success());
-#else
-  std::unique_ptr<Error> Err = llvm::make_unique<Error>(Error::success());
-#endif
   auto Cur = Archive->child_begin(*Err);
   if (*Err) {
     LLVMRustSetLastError(toString(std::move(*Err)).c_str());
diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
index e97d96e3a4e..35cca04b20f 100644
--- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
@@ -23,10 +23,17 @@ extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer(
     const char* const Filenames[],
     size_t FilenamesLen,
     RustStringRef BufferOut) {
+#if LLVM_VERSION_GE(13,0)
+  SmallVector<std::string,32> FilenameRefs;
+  for (size_t i = 0; i < FilenamesLen; i++) {
+    FilenameRefs.push_back(std::string(Filenames[i]));
+  }
+#else
   SmallVector<StringRef,32> FilenameRefs;
   for (size_t i = 0; i < FilenamesLen; i++) {
     FilenameRefs.push_back(StringRef(Filenames[i]));
   }
+#endif
   auto FilenamesWriter = coverage::CoverageFilenamesSectionWriter(
     makeArrayRef(FilenameRefs));
   RawRustStringOstream OS(BufferOut);
diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
index f67e06706ea..0b1b68d83b7 100644
--- a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
+++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
@@ -33,13 +33,6 @@
   (LLVM_VERSION_MAJOR > (major) ||                                             \
    LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR >= (minor))
 
-#define LLVM_VERSION_EQ(major, minor)                                          \
-  (LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR == (minor))
-
-#define LLVM_VERSION_LE(major, minor)                                          \
-  (LLVM_VERSION_MAJOR < (major) ||                                             \
-   LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR <= (minor))
-
 #define LLVM_VERSION_LT(major, minor) (!LLVM_VERSION_GE((major), (minor)))
 
 #include "llvm/IR/LegacyPassManager.h"
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
index 2f28162f908..617b2ed970e 100644
--- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
@@ -65,13 +65,9 @@ extern "C" void LLVMInitializePasses() {
 }
 
 extern "C" void LLVMTimeTraceProfilerInitialize() {
-#if LLVM_VERSION_GE(10, 0)
   timeTraceProfilerInitialize(
       /* TimeTraceGranularity */ 0,
       /* ProcName */ "rustc");
-#else
-  timeTraceProfilerInitialize();
-#endif
 }
 
 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
@@ -408,26 +404,21 @@ extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
   printf("\n");
 }
 
-extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
+extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
   const TargetMachine *Target = unwrap(TM);
   const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
   const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
-  unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
-
-  printf("Available features for this target:\n");
-  for (auto &Feature : FeatTable)
-    printf("    %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
-  printf("\nRust-specific features:\n");
-  printf("    %-*s - %s.\n",
-    MaxFeatLen,
-    "crt-static",
-    "Enables libraries with C Run-time Libraries(CRT) to be statically linked"
-  );
-  printf("\n");
+  return FeatTable.size();
+}
 
-  printf("Use +feature to enable a feature, or -feature to disable it.\n"
-         "For example, rustc -C -target-cpu=mycpu -C "
-         "target-feature=+feature1,-feature2\n\n");
+extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
+                                         const char** Feature, const char** Desc) {
+  const TargetMachine *Target = unwrap(TM);
+  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
+  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
+  const SubtargetFeatureKV Feat = FeatTable[Index];
+  *Feature = Feat.Key;
+  *Desc = Feat.Desc;
 }
 
 #else
@@ -436,9 +427,11 @@ extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
   printf("Target CPU help is not supported by this LLVM version.\n\n");
 }
 
-extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
-  printf("Target features help is not supported by this LLVM version.\n\n");
+extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
+  return 0;
 }
+
+extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
 #endif
 
 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
@@ -596,7 +589,6 @@ enum class LLVMRustFileType {
   ObjectFile,
 };
 
-#if LLVM_VERSION_GE(10, 0)
 static CodeGenFileType fromRust(LLVMRustFileType Type) {
   switch (Type) {
   case LLVMRustFileType::AssemblyFile:
@@ -607,18 +599,6 @@ static CodeGenFileType fromRust(LLVMRustFileType Type) {
     report_fatal_error("Bad FileType.");
   }
 }
-#else
-static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
-  switch (Type) {
-  case LLVMRustFileType::AssemblyFile:
-    return TargetMachine::CGFT_AssemblyFile;
-  case LLVMRustFileType::ObjectFile:
-    return TargetMachine::CGFT_ObjectFile;
-  default:
-    report_fatal_error("Bad FileType.");
-  }
-}
-#endif
 
 extern "C" LLVMRustResult
 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
@@ -772,14 +752,18 @@ LLVMRustOptimizeWithNewPassManager(
   TargetMachine *TM = unwrap(TMRef);
   PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
 
-  // FIXME: MergeFunctions is not supported by NewPM yet.
-  (void) MergeFunctions;
 
   PipelineTuningOptions PTO;
   PTO.LoopUnrolling = UnrollLoops;
   PTO.LoopInterleaving = UnrollLoops;
   PTO.LoopVectorization = LoopVectorize;
   PTO.SLPVectorization = SLPVectorize;
+#if LLVM_VERSION_GE(12, 0)
+  PTO.MergeFunctions = MergeFunctions;
+#else
+  // MergeFunctions is not supported by NewPM in older LLVM versions.
+  (void) MergeFunctions;
+#endif
 
   // FIXME: We may want to expose this as an option.
   bool DebugPassManager = false;
@@ -864,13 +848,11 @@ LLVMRustOptimizeWithNewPassManager(
         }
       );
 #else
-#if LLVM_VERSION_GE(10, 0)
       PipelineStartEPCallbacks.push_back(
         [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
           MPM.addPass(MemorySanitizerPass(Options));
         }
       );
-#endif
       OptimizerLastEPCallbacks.push_back(
         [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
           FPM.addPass(MemorySanitizerPass(Options));
@@ -888,13 +870,11 @@ LLVMRustOptimizeWithNewPassManager(
         }
       );
 #else
-#if LLVM_VERSION_GE(10, 0)
       PipelineStartEPCallbacks.push_back(
         [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
           MPM.addPass(ThreadSanitizerPass());
         }
       );
-#endif
       OptimizerLastEPCallbacks.push_back(
         [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
           FPM.addPass(ThreadSanitizerPass());
@@ -985,13 +965,11 @@ LLVMRustOptimizeWithNewPassManager(
 
       MPM.addPass(AlwaysInlinerPass(EmitLifetimeMarkers));
 
-# if LLVM_VERSION_GE(10, 0)
       if (PGOOpt) {
         PB.addPGOInstrPassesForO0(
             MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
             /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
       }
-# endif
 #endif
     } else {
 #if LLVM_VERSION_GE(12, 0)
@@ -1260,6 +1238,14 @@ extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
   unwrap(M)->setPIELevel(PIELevel::Level::Large);
 }
 
+extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
+                                           LLVMRustCodeModel Model) {
+  auto CM = fromRust(Model);
+  if (!CM.hasValue())
+    return;
+  unwrap(M)->setCodeModel(*CM);
+}
+
 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
 // right now. This ThinLTO support is only enabled on "recent ish" versions of
 // LLVM, and otherwise it's just blanket rejected from other compilers.
@@ -1354,11 +1340,7 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
                           int num_modules,
                           const char **preserved_symbols,
                           int num_symbols) {
-#if LLVM_VERSION_GE(10, 0)
   auto Ret = std::make_unique<LLVMRustThinLTOData>();
-#else
-  auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
-#endif
 
   // Load each module's summary and merge it into one combined index
   for (int i = 0; i < num_modules; i++) {
@@ -1425,9 +1407,17 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
     Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
   };
 
+#if LLVM_VERSION_GE(13,0)
+  // Uses FromPrevailing visibility scheme which works for many binary
+  // formats. We probably could and should use ELF visibility scheme for many of
+  // our targets, however.
+  lto::Config conf;
+  thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
+                                  Ret->GUIDPreservedSymbols);
+#else
   thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
                                   Ret->GUIDPreservedSymbols);
-
+#endif
   // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
   // callback below. This callback below will dictate the linkage for all
   // summaries in the index, and we basically just only want to ensure that dead
@@ -1443,7 +1433,6 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
         ExportedGUIDs.insert(GUID);
     }
   }
-#if LLVM_VERSION_GE(10, 0)
   auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
     const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
     return (ExportList != Ret->ExportLists.end() &&
@@ -1451,15 +1440,6 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
       ExportedGUIDs.count(VI.getGUID());
   };
   thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
-#else
-  auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
-    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
-    return (ExportList != Ret->ExportLists.end() &&
-      ExportList->second.count(GUID)) ||
-      ExportedGUIDs.count(GUID);
-  };
-  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
-#endif
 
   return Ret.release();
 }
@@ -1616,11 +1596,7 @@ struct LLVMRustThinLTOBuffer {
 
 extern "C" LLVMRustThinLTOBuffer*
 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
-#if LLVM_VERSION_GE(10, 0)
   auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
-#else
-  auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
-#endif
   {
     raw_string_ostream OS(Ret->data);
     {
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index a8536595404..2e135fbe2bd 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -349,8 +349,10 @@ extern "C" void LLVMRustRemoveFunctionAttributes(LLVMValueRef Fn,
   F->setAttributes(PALNew);
 }
 
-// enable fpmath flag UnsafeAlgebra
-extern "C" void LLVMRustSetHasUnsafeAlgebra(LLVMValueRef V) {
+// Enable a fast-math flag
+//
+// https://llvm.org/docs/LangRef.html#fast-math-flags
+extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
   if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
     I->setFast(true);
   }
@@ -382,9 +384,18 @@ LLVMRustBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Target,
                            LLVMValueRef Old, LLVMValueRef Source,
                            LLVMAtomicOrdering Order,
                            LLVMAtomicOrdering FailureOrder, LLVMBool Weak) {
+#if LLVM_VERSION_GE(13,0)
+  // Rust probably knows the alignment of the target value and should be able to
+  // specify something more precise than MaybeAlign here. See also
+  // https://reviews.llvm.org/D97224 which may be a useful reference.
+  AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg(
+      unwrap(Target), unwrap(Old), unwrap(Source), llvm::MaybeAlign(), fromRust(Order),
+      fromRust(FailureOrder));
+#else
   AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg(
       unwrap(Target), unwrap(Old), unwrap(Source), fromRust(Order),
       fromRust(FailureOrder));
+#endif
   ACXI->setWeak(Weak);
   return wrap(ACXI);
 }
@@ -532,11 +543,6 @@ static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) {
   if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) {
     Result |= DINode::DIFlags::FlagAppleBlock;
   }
-#if LLVM_VERSION_LT(10, 0)
-  if (isSet(Flags & LLVMRustDIFlags::FlagBlockByrefStruct)) {
-    Result |= DINode::DIFlags::FlagBlockByrefStruct;
-  }
-#endif
   if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) {
     Result |= DINode::DIFlags::FlagVirtual;
   }
@@ -901,9 +907,7 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable(
       unwrapDI<DIDescriptor>(Context), StringRef(Name, NameLen),
       StringRef(LinkageName, LinkageNameLen),
       unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), IsLocalToUnit,
-#if LLVM_VERSION_GE(10, 0)
       /* isDefined */ true,
-#endif
       InitExpr, unwrapDIPtr<MDNode>(Decl),
       /* templateParams */ nullptr,
       AlignInBits);
@@ -1090,19 +1094,11 @@ inline section_iterator *unwrap(LLVMSectionIteratorRef SI) {
 
 extern "C" size_t LLVMRustGetSectionName(LLVMSectionIteratorRef SI,
                                          const char **Ptr) {
-#if LLVM_VERSION_GE(10, 0)
   auto NameOrErr = (*unwrap(SI))->getName();
   if (!NameOrErr)
     report_fatal_error(NameOrErr.takeError());
   *Ptr = NameOrErr->data();
   return NameOrErr->size();
-#else
-  StringRef Ret;
-  if (std::error_code EC = (*unwrap(SI))->getName(Ret))
-    report_fatal_error(EC.message());
-  *Ptr = Ret.data();
-  return Ret.size();
-#endif
 }
 
 // LLVMArrayType function does not support 64-bit ElementCount
@@ -1304,9 +1300,19 @@ extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) {
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef)
 
+#if LLVM_VERSION_LT(13, 0)
+using LLVMInlineAsmDiagHandlerTy = LLVMContext::InlineAsmDiagHandlerTy;
+#else
+using LLVMInlineAsmDiagHandlerTy = void*;
+#endif
+
 extern "C" void LLVMRustSetInlineAsmDiagnosticHandler(
-    LLVMContextRef C, LLVMContext::InlineAsmDiagHandlerTy H, void *CX) {
+    LLVMContextRef C, LLVMInlineAsmDiagHandlerTy H, void *CX) {
+  // Diagnostic handlers were unified in LLVM change 5de2d189e6ad, so starting
+  // with LLVM 13 this function is gone.
+#if LLVM_VERSION_LT(13, 0)
   unwrap(C)->setInlineAsmDiagnosticHandler(H, CX);
+#endif
 }
 
 extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef,
@@ -1441,47 +1447,28 @@ extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B,
                                             LLVMValueRef Dst, unsigned DstAlign,
                                             LLVMValueRef Src, unsigned SrcAlign,
                                             LLVMValueRef Size, bool IsVolatile) {
-#if LLVM_VERSION_GE(10, 0)
   return wrap(unwrap(B)->CreateMemCpy(
       unwrap(Dst), MaybeAlign(DstAlign),
       unwrap(Src), MaybeAlign(SrcAlign),
       unwrap(Size), IsVolatile));
-#else
-  return wrap(unwrap(B)->CreateMemCpy(
-      unwrap(Dst), DstAlign,
-      unwrap(Src), SrcAlign,
-      unwrap(Size), IsVolatile));
-#endif
 }
 
 extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B,
                                              LLVMValueRef Dst, unsigned DstAlign,
                                              LLVMValueRef Src, unsigned SrcAlign,
                                              LLVMValueRef Size, bool IsVolatile) {
-#if LLVM_VERSION_GE(10, 0)
   return wrap(unwrap(B)->CreateMemMove(
       unwrap(Dst), MaybeAlign(DstAlign),
       unwrap(Src), MaybeAlign(SrcAlign),
       unwrap(Size), IsVolatile));
-#else
-  return wrap(unwrap(B)->CreateMemMove(
-      unwrap(Dst), DstAlign,
-      unwrap(Src), SrcAlign,
-      unwrap(Size), IsVolatile));
-#endif
 }
 
 extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B,
                                             LLVMValueRef Dst, unsigned DstAlign,
                                             LLVMValueRef Val,
                                             LLVMValueRef Size, bool IsVolatile) {
-#if LLVM_VERSION_GE(10, 0)
   return wrap(unwrap(B)->CreateMemSet(
       unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile));
-#else
-  return wrap(unwrap(B)->CreateMemSet(
-      unwrap(Dst), unwrap(Val), unwrap(Size), DstAlign, IsVolatile));
-#endif
 }
 
 extern "C" LLVMValueRef
@@ -1661,17 +1648,17 @@ extern "C" void LLVMRustSetVisibility(LLVMValueRef V,
   LLVMSetVisibility(V, fromRust(RustVisibility));
 }
 
+extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) {
+  unwrap<GlobalValue>(Global)->setDSOLocal(is_dso_local);
+}
+
 struct LLVMRustModuleBuffer {
   std::string data;
 };
 
 extern "C" LLVMRustModuleBuffer*
 LLVMRustModuleBufferCreate(LLVMModuleRef M) {
-#if LLVM_VERSION_GE(10, 0)
   auto Ret = std::make_unique<LLVMRustModuleBuffer>();
-#else
-  auto Ret = llvm::make_unique<LLVMRustModuleBuffer>();
-#endif
   {
     raw_string_ostream OS(Ret->data);
     {
diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs
index 592010d78cf..555aefb1929 100644
--- a/compiler/rustc_llvm/src/lib.rs
+++ b/compiler/rustc_llvm/src/lib.rs
@@ -125,10 +125,7 @@ pub fn initialize_available_targets() {
         LLVMInitializeMSP430TargetInfo,
         LLVMInitializeMSP430Target,
         LLVMInitializeMSP430TargetMC,
-        LLVMInitializeMSP430AsmPrinter
-    );
-    init_target!(
-        all(llvm_component = "msp430", llvm_has_msp430_asm_parser),
+        LLVMInitializeMSP430AsmPrinter,
         LLVMInitializeMSP430AsmParser
     );
     init_target!(
diff --git a/compiler/rustc_macros/src/hash_stable.rs b/compiler/rustc_macros/src/hash_stable.rs
index c955c137782..30569f20793 100644
--- a/compiler/rustc_macros/src/hash_stable.rs
+++ b/compiler/rustc_macros/src/hash_stable.rs
@@ -74,6 +74,7 @@ pub fn hash_stable_generic_derive(mut s: synstructure::Structure<'_>) -> proc_ma
     s.bound_impl(
         quote!(::rustc_data_structures::stable_hasher::HashStable<__CTX>),
         quote! {
+            #[inline]
             fn hash_stable(
                 &self,
                 __hcx: &mut __CTX,
@@ -119,6 +120,7 @@ pub fn hash_stable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::To
             >
         ),
         quote! {
+            #[inline]
             fn hash_stable(
                 &self,
                 __hcx: &mut ::rustc_middle::ich::StableHashingContext<'__ctx>,
diff --git a/compiler/rustc_macros/src/symbols/tests.rs b/compiler/rustc_macros/src/symbols/tests.rs
index 82b4b876978..842d2a97718 100644
--- a/compiler/rustc_macros/src/symbols/tests.rs
+++ b/compiler/rustc_macros/src/symbols/tests.rs
@@ -43,7 +43,7 @@ fn test_symbols_macro(input: TokenStream, expected_errors: &[&str]) {
         "Macro generated a different number of errors than expected"
     );
 
-    for (found_error, &expected_error) in found_errors.iter().zip(expected_errors.iter()) {
+    for (found_error, &expected_error) in found_errors.iter().zip(expected_errors) {
         let found_error_str = format!("{}", found_error);
         assert_eq!(found_error_str, expected_error);
     }
diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml
index 2aabc2c407b..29fa0b70069 100644
--- a/compiler/rustc_metadata/Cargo.toml
+++ b/compiler/rustc_metadata/Cargo.toml
@@ -11,7 +11,6 @@ doctest = false
 libc = "0.2"
 snap = "1"
 tracing = "0.1"
-memmap = "0.7"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_middle = { path = "../rustc_middle" }
 rustc_attr = { path = "../rustc_attr" }
diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs
index 63c6f369eb6..26db3a5f39d 100644
--- a/compiler/rustc_metadata/src/creader.rs
+++ b/compiler/rustc_metadata/src/creader.rs
@@ -6,11 +6,11 @@ use crate::rmeta::{CrateDep, CrateMetadata, CrateNumMap, CrateRoot, MetadataBlob
 
 use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_ast::{self as ast, *};
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::Lrc;
 use rustc_expand::base::SyntaxExtension;
-use rustc_hir::def_id::{CrateNum, LocalDefId, LOCAL_CRATE};
+use rustc_hir::def_id::{CrateNum, LocalDefId, StableCrateId, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_index::vec::IndexVec;
 use rustc_middle::middle::cstore::{CrateDepKind, CrateSource, ExternCrate};
@@ -42,6 +42,13 @@ pub struct CStore {
     allocator_kind: Option<AllocatorKind>,
     /// This crate has a `#[global_allocator]` item.
     has_global_allocator: bool,
+
+    /// This map is used to verify we get no hash conflicts between
+    /// `StableCrateId` values.
+    stable_crate_ids: FxHashMap<StableCrateId, CrateNum>,
+
+    /// Unused externs of the crate
+    unused_externs: Vec<Symbol>,
 }
 
 pub struct CrateLoader<'a> {
@@ -186,6 +193,27 @@ impl CStore {
     crate fn has_global_allocator(&self) -> bool {
         self.has_global_allocator
     }
+
+    pub fn report_unused_deps(&self, tcx: TyCtxt<'_>) {
+        // We put the check for the option before the lint_level_at_node call
+        // because the call mutates internal state and introducing it
+        // leads to some ui tests failing.
+        if !tcx.sess.opts.json_unused_externs {
+            return;
+        }
+        let level = tcx
+            .lint_level_at_node(lint::builtin::UNUSED_CRATE_DEPENDENCIES, rustc_hir::CRATE_HIR_ID)
+            .0;
+        if level != lint::Level::Allow {
+            let unused_externs =
+                self.unused_externs.iter().map(|ident| ident.to_ident_string()).collect::<Vec<_>>();
+            let unused_externs = unused_externs.iter().map(String::as_str).collect::<Vec<&str>>();
+            tcx.sess
+                .parse_sess
+                .span_diagnostic
+                .emit_unused_externs(level.as_str(), &unused_externs);
+        }
+    }
 }
 
 impl<'a> CrateLoader<'a> {
@@ -194,6 +222,11 @@ impl<'a> CrateLoader<'a> {
         metadata_loader: &'a MetadataLoaderDyn,
         local_crate_name: &str,
     ) -> Self {
+        let local_crate_stable_id =
+            StableCrateId::new(local_crate_name, sess.local_crate_disambiguator());
+        let mut stable_crate_ids = FxHashMap::default();
+        stable_crate_ids.insert(local_crate_stable_id, LOCAL_CRATE);
+
         CrateLoader {
             sess,
             metadata_loader,
@@ -207,6 +240,8 @@ impl<'a> CrateLoader<'a> {
                 injected_panic_runtime: None,
                 allocator_kind: None,
                 has_global_allocator: false,
+                stable_crate_ids,
+                unused_externs: Vec::new(),
             },
             used_extern_options: Default::default(),
         }
@@ -313,6 +348,20 @@ impl<'a> CrateLoader<'a> {
         res
     }
 
+    fn verify_no_stable_crate_id_hash_conflicts(
+        &mut self,
+        root: &CrateRoot<'_>,
+        cnum: CrateNum,
+    ) -> Result<(), CrateError> {
+        if let Some(existing) = self.cstore.stable_crate_ids.insert(root.stable_crate_id(), cnum) {
+            let crate_name0 = root.name();
+            let crate_name1 = self.cstore.get_crate_data(existing).name();
+            return Err(CrateError::StableCrateIdCollision(crate_name0, crate_name1));
+        }
+
+        Ok(())
+    }
+
     fn register_crate(
         &mut self,
         host_lib: Option<Library>,
@@ -326,7 +375,6 @@ impl<'a> CrateLoader<'a> {
         let Library { source, metadata } = lib;
         let crate_root = metadata.get_root();
         let host_hash = host_lib.as_ref().map(|lib| lib.metadata.get_root().hash());
-        self.verify_no_symbol_conflicts(&crate_root)?;
 
         let private_dep =
             self.sess.opts.externs.get(&name.as_str()).map_or(false, |e| e.is_private_dep);
@@ -368,6 +416,14 @@ impl<'a> CrateLoader<'a> {
             None
         };
 
+        // Perform some verification *after* resolve_crate_deps() above is
+        // known to have been successful. It seems that - in error cases - the
+        // cstore can be in a temporarily invalid state between cnum allocation
+        // and dependency resolution and the verification code would produce
+        // ICEs in that case (see #83045).
+        self.verify_no_symbol_conflicts(&crate_root)?;
+        self.verify_no_stable_crate_id_hash_conflicts(&crate_root, cnum)?;
+
         let crate_metadata = CrateMetadata::new(
             self.sess,
             metadata,
@@ -710,7 +766,7 @@ impl<'a> CrateLoader<'a> {
     }
 
     fn inject_profiler_runtime(&mut self, krate: &ast::Crate) {
-        if (self.sess.opts.debugging_opts.instrument_coverage
+        if (self.sess.instrument_coverage()
             || self.sess.opts.debugging_opts.profile
             || self.sess.opts.cg.profile_generate.enabled())
             && !self.sess.opts.debugging_opts.no_profiler_runtime
@@ -873,11 +929,17 @@ impl<'a> CrateLoader<'a> {
                 // Don't worry about pathless `--extern foo` sysroot references
                 continue;
             }
-            if self.used_extern_options.contains(&Symbol::intern(name)) {
+            let name_interned = Symbol::intern(name);
+            if self.used_extern_options.contains(&name_interned) {
                 continue;
             }
 
             // Got a real unused --extern
+            if self.sess.opts.json_unused_externs {
+                self.cstore.unused_externs.push(name_interned);
+                continue;
+            }
+
             let diag = match self.sess.opts.extern_dep_specs.get(name) {
                 Some(loc) => BuiltinLintDiagnostics::ExternDepSpec(name.clone(), loc.into()),
                 None => {
@@ -910,9 +972,9 @@ impl<'a> CrateLoader<'a> {
         self.inject_allocator_crate(krate);
         self.inject_panic_runtime(krate);
 
-        info!("{:?}", CrateDump(&self.cstore));
-
         self.report_unused_deps(krate);
+
+        info!("{:?}", CrateDump(&self.cstore));
     }
 
     pub fn process_extern_crate(
diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs
index 2560cfa7462..c4d9e3f77f0 100644
--- a/compiler/rustc_metadata/src/lib.rs
+++ b/compiler/rustc_metadata/src/lib.rs
@@ -5,7 +5,7 @@
 #![feature(in_band_lifetimes)]
 #![feature(nll)]
 #![feature(once_cell)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(proc_macro_internals)]
 #![feature(min_specialization)]
 #![feature(stmt_expr_attributes)]
@@ -26,7 +26,6 @@ pub use rmeta::{provide, provide_extern};
 
 mod dependency_format;
 mod foreign_modules;
-mod link_args;
 mod native_libs;
 mod rmeta;
 
diff --git a/compiler/rustc_metadata/src/link_args.rs b/compiler/rustc_metadata/src/link_args.rs
deleted file mode 100644
index d088288c507..00000000000
--- a/compiler/rustc_metadata/src/link_args.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-use rustc_hir as hir;
-use rustc_hir::itemlikevisit::ItemLikeVisitor;
-use rustc_middle::ty::TyCtxt;
-use rustc_span::symbol::{sym, Symbol};
-use rustc_target::spec::abi::Abi;
-
-crate fn collect(tcx: TyCtxt<'_>) -> Vec<String> {
-    let mut collector = Collector { tcx, args: Vec::new() };
-    tcx.hir().krate().visit_all_item_likes(&mut collector);
-
-    for attr in tcx.hir().krate().item.attrs.iter() {
-        if attr.has_name(sym::link_args) {
-            if let Some(linkarg) = attr.value_str() {
-                collector.add_link_args(linkarg);
-            }
-        }
-    }
-
-    collector.args
-}
-
-struct Collector<'tcx> {
-    tcx: TyCtxt<'tcx>,
-    args: Vec<String>,
-}
-
-impl<'tcx> ItemLikeVisitor<'tcx> for Collector<'tcx> {
-    fn visit_item(&mut self, it: &'tcx hir::Item<'tcx>) {
-        let abi = match it.kind {
-            hir::ItemKind::ForeignMod { abi, .. } => abi,
-            _ => return,
-        };
-        if abi == Abi::Rust || abi == Abi::RustIntrinsic || abi == Abi::PlatformIntrinsic {
-            return;
-        }
-
-        // First, add all of the custom #[link_args] attributes
-        let sess = &self.tcx.sess;
-        for m in it.attrs.iter().filter(|a| sess.check_name(a, sym::link_args)) {
-            if let Some(linkarg) = m.value_str() {
-                self.add_link_args(linkarg);
-            }
-        }
-    }
-
-    fn visit_trait_item(&mut self, _it: &'tcx hir::TraitItem<'tcx>) {}
-    fn visit_impl_item(&mut self, _it: &'tcx hir::ImplItem<'tcx>) {}
-    fn visit_foreign_item(&mut self, _it: &'tcx hir::ForeignItem<'tcx>) {}
-}
-
-impl<'tcx> Collector<'tcx> {
-    fn add_link_args(&mut self, args: Symbol) {
-        self.args.extend(args.as_str().split(' ').filter(|s| !s.is_empty()).map(|s| s.to_string()))
-    }
-}
diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs
index b66c6cffb1b..7f6311861c1 100644
--- a/compiler/rustc_metadata/src/locator.rs
+++ b/compiler/rustc_metadata/src/locator.rs
@@ -216,6 +216,7 @@ use crate::creader::Library;
 use crate::rmeta::{rustc_version, MetadataBlob, METADATA_HEADER};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::owning_ref::OwningRef;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
@@ -232,7 +233,6 @@ use rustc_target::spec::{Target, TargetTriple};
 
 use snap::read::FrameDecoder;
 use std::io::{Read, Result as IoResult, Write};
-use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::{cmp, fmt, fs};
 use tracing::{debug, info, warn};
@@ -727,19 +727,6 @@ impl<'a> CrateLocator<'a> {
     }
 }
 
-/// A trivial wrapper for `Mmap` that implements `StableDeref`.
-struct StableDerefMmap(memmap::Mmap);
-
-impl Deref for StableDerefMmap {
-    type Target = [u8];
-
-    fn deref(&self) -> &[u8] {
-        self.0.deref()
-    }
-}
-
-unsafe impl stable_deref_trait::StableDeref for StableDerefMmap {}
-
 fn get_metadata_section(
     target: &Target,
     flavor: CrateFlavor,
@@ -779,11 +766,11 @@ fn get_metadata_section(
             // mmap the file, because only a small fraction of it is read.
             let file = std::fs::File::open(filename)
                 .map_err(|_| format!("failed to open rmeta metadata: '{}'", filename.display()))?;
-            let mmap = unsafe { memmap::Mmap::map(&file) };
+            let mmap = unsafe { Mmap::map(file) };
             let mmap = mmap
                 .map_err(|_| format!("failed to mmap rmeta metadata: '{}'", filename.display()))?;
 
-            rustc_erase_owner!(OwningRef::new(StableDerefMmap(mmap)).map_owner_box())
+            rustc_erase_owner!(OwningRef::new(mmap).map_owner_box())
         }
     };
     let blob = MetadataBlob::new(raw_bytes);
@@ -888,6 +875,7 @@ crate enum CrateError {
     MultipleMatchingCrates(Symbol, FxHashMap<Svh, Library>),
     SymbolConflictsCurrent(Symbol),
     SymbolConflictsOthers(Symbol),
+    StableCrateIdCollision(Symbol, Symbol),
     DlOpen(String),
     DlSym(String),
     LocatorCombined(CombinedLocatorError),
@@ -970,6 +958,13 @@ impl CrateError {
                  `-C metadata`. This will result in symbol conflicts between the two.",
                 root_name,
             ),
+            CrateError::StableCrateIdCollision(crate_name0, crate_name1) => {
+                let msg = format!(
+                    "found crates (`{}` and `{}`) with colliding StableCrateId values.",
+                    crate_name0, crate_name1
+                );
+                sess.struct_span_err(span, &msg)
+            }
             CrateError::DlOpen(s) | CrateError::DlSym(s) => sess.struct_span_err(span, &s),
             CrateError::LocatorCombined(locator) => {
                 let crate_name = locator.crate_name;
diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs
index 4d63b6d074a..523e016eeb9 100644
--- a/compiler/rustc_metadata/src/native_libs.rs
+++ b/compiler/rustc_metadata/src/native_libs.rs
@@ -44,7 +44,8 @@ impl ItemLikeVisitor<'tcx> for Collector<'tcx> {
 
         // Process all of the #[link(..)]-style arguments
         let sess = &self.tcx.sess;
-        for m in it.attrs.iter().filter(|a| sess.check_name(a, sym::link)) {
+        for m in self.tcx.hir().attrs(it.hir_id()).iter().filter(|a| sess.check_name(a, sym::link))
+        {
             let items = match m.meta_item_list() {
                 Some(item) => item,
                 None => continue,
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs
index e3c35390798..3d0a9d553b0 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder.rs
@@ -7,7 +7,6 @@ use crate::rmeta::*;
 use rustc_ast as ast;
 use rustc_attr as attr;
 use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fingerprint::{Fingerprint, FingerprintDecoder};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::{Lock, LockGuard, Lrc, OnceCell};
@@ -351,12 +350,6 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for DefIndex {
     }
 }
 
-impl<'a, 'tcx> FingerprintDecoder for DecodeContext<'a, 'tcx> {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, String> {
-        Fingerprint::decode_opaque(&mut self.opaque)
-    }
-}
-
 impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for SyntaxContext {
     fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Result<SyntaxContext, String> {
         let cdata = decoder.cdata();
@@ -635,6 +628,10 @@ impl CrateRoot<'_> {
         self.hash
     }
 
+    crate fn stable_crate_id(&self) -> StableCrateId {
+        self.stable_crate_id
+    }
+
     crate fn triple(&self) -> &TargetTriple {
         &self.triple
     }
@@ -956,6 +953,14 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         self.root.tables.expn_that_defined.get(self, id).unwrap().decode((self, sess))
     }
 
+    fn get_const_param_default(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        id: DefIndex,
+    ) -> rustc_middle::ty::Const<'tcx> {
+        self.root.tables.const_defaults.get(self, id).unwrap().decode((self, tcx))
+    }
+
     /// Iterates over all the stability attributes in the given crate.
     fn get_lib_features(&self, tcx: TyCtxt<'tcx>) -> &'tcx [(Symbol, Option<Symbol>)] {
         // FIXME: For a proc macro crate, not sure whether we should return the "host"
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index 0f860d11dc2..bebee9dac3b 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -1,6 +1,5 @@
 use crate::creader::{CStore, LoadedMacro};
 use crate::foreign_modules;
-use crate::link_args;
 use crate::native_libs;
 use crate::rmeta::{self, encoder};
 
@@ -122,6 +121,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
     promoted_mir => { tcx.arena.alloc(cdata.get_promoted_mir(tcx, def_id.index)) }
     mir_abstract_const => { cdata.get_mir_abstract_const(tcx, def_id.index) }
     unused_generic_params => { cdata.get_unused_generic_params(def_id.index) }
+    const_param_default => { tcx.mk_const(cdata.get_const_param_default(tcx, def_id.index)) }
     mir_const_qualif => { cdata.mir_const_qualif(def_id.index) }
     fn_sig => { cdata.fn_sig(def_id.index, tcx) }
     inherent_impls => { cdata.get_inherent_implementations_for_type(tcx, def_id.index) }
@@ -294,10 +294,6 @@ pub fn provide(providers: &mut Providers) {
                 foreign_modules::collect(tcx).into_iter().map(|m| (m.def_id, m)).collect();
             Lrc::new(modules)
         },
-        link_args: |tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
-            Lrc::new(link_args::collect(tcx))
-        },
 
         // Returns a map from a sufficiently visible external item (i.e., an
         // external item that is visible from at least one local module) to a
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 61265d7204c..a5157854e15 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -1,7 +1,6 @@
 use crate::rmeta::table::{FixedSizeEncoding, TableBuilder};
 use crate::rmeta::*;
 
-use rustc_data_structures::fingerprint::{Fingerprint, FingerprintEncoder};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::sync::{join, par_iter, Lrc, ParallelIterator};
@@ -116,6 +115,7 @@ impl<'a, 'tcx> Encoder for EncodeContext<'a, 'tcx> {
         emit_f32(f32);
         emit_char(char);
         emit_str(&str);
+        emit_raw_bytes(&[u8]);
     }
 }
 
@@ -307,12 +307,6 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for Span {
     }
 }
 
-impl<'a, 'tcx> FingerprintEncoder for EncodeContext<'a, 'tcx> {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> Result<(), Self::Error> {
-        self.opaque.encode_fingerprint(f)
-    }
-}
-
 impl<'a, 'tcx> TyEncoder<'tcx> for EncodeContext<'a, 'tcx> {
     const CLEAR_CROSS_CRATE: bool = true;
 
@@ -433,7 +427,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
     fn encode_info_for_items(&mut self) {
         let krate = self.tcx.hir().krate();
-        self.encode_info_for_mod(CRATE_DEF_ID, &krate.item.module);
+        self.encode_info_for_mod(CRATE_DEF_ID, &krate.item);
 
         // Proc-macro crates only export proc-macro items, which are looked
         // up using `proc_macro_data`
@@ -653,6 +647,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             triple: tcx.sess.opts.target_triple.clone(),
             hash: tcx.crate_hash(LOCAL_CRATE),
             disambiguator: tcx.sess.local_crate_disambiguator(),
+            stable_crate_id: tcx.def_path_hash(LOCAL_CRATE.as_def_id()).stable_crate_id(),
             panic_strategy: tcx.sess.panic_strategy(),
             edition: tcx.sess.edition(),
             has_global_allocator: tcx.has_global_allocator(LOCAL_CRATE),
@@ -835,6 +830,76 @@ fn should_encode_mir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> (bool, bool) {
     }
 }
 
+fn should_encode_variances(def_kind: DefKind) -> bool {
+    match def_kind {
+        DefKind::Struct
+        | DefKind::Union
+        | DefKind::Enum
+        | DefKind::Variant
+        | DefKind::Fn
+        | DefKind::Ctor(..)
+        | DefKind::AssocFn => true,
+        DefKind::Mod
+        | DefKind::Field
+        | DefKind::AssocTy
+        | DefKind::AssocConst
+        | DefKind::TyParam
+        | DefKind::ConstParam
+        | DefKind::Static
+        | DefKind::Const
+        | DefKind::ForeignMod
+        | DefKind::TyAlias
+        | DefKind::OpaqueTy
+        | DefKind::Impl
+        | DefKind::Trait
+        | DefKind::TraitAlias
+        | DefKind::Macro(..)
+        | DefKind::ForeignTy
+        | DefKind::Use
+        | DefKind::LifetimeParam
+        | DefKind::AnonConst
+        | DefKind::GlobalAsm
+        | DefKind::Closure
+        | DefKind::Generator
+        | DefKind::ExternCrate => false,
+    }
+}
+
+fn should_encode_generics(def_kind: DefKind) -> bool {
+    match def_kind {
+        DefKind::Struct
+        | DefKind::Union
+        | DefKind::Enum
+        | DefKind::Variant
+        | DefKind::Trait
+        | DefKind::TyAlias
+        | DefKind::ForeignTy
+        | DefKind::TraitAlias
+        | DefKind::AssocTy
+        | DefKind::Fn
+        | DefKind::Const
+        | DefKind::Static
+        | DefKind::Ctor(..)
+        | DefKind::AssocFn
+        | DefKind::AssocConst
+        | DefKind::AnonConst
+        | DefKind::OpaqueTy
+        | DefKind::Impl
+        | DefKind::Closure
+        | DefKind::Generator => true,
+        DefKind::Mod
+        | DefKind::Field
+        | DefKind::ForeignMod
+        | DefKind::TyParam
+        | DefKind::ConstParam
+        | DefKind::Macro(..)
+        | DefKind::Use
+        | DefKind::LifetimeParam
+        | DefKind::GlobalAsm
+        | DefKind::ExternCrate => false,
+    }
+}
+
 impl EncodeContext<'a, 'tcx> {
     fn encode_def_ids(&mut self) {
         if self.is_proc_macro {
@@ -863,12 +928,34 @@ impl EncodeContext<'a, 'tcx> {
                 self.encode_const_stability(def_id);
                 self.encode_deprecation(def_id);
             }
+            if should_encode_variances(def_kind) {
+                let v = self.tcx.variances_of(def_id);
+                record!(self.tables.variances[def_id] <- v);
+            }
+            if should_encode_generics(def_kind) {
+                let g = tcx.generics_of(def_id);
+                record!(self.tables.generics[def_id] <- g);
+                record!(self.tables.explicit_predicates[def_id] <- self.tcx.explicit_predicates_of(def_id));
+                let inferred_outlives = self.tcx.inferred_outlives_of(def_id);
+                if !inferred_outlives.is_empty() {
+                    record!(self.tables.inferred_outlives[def_id] <- inferred_outlives);
+                }
+            }
+            if let DefKind::Trait | DefKind::TraitAlias = def_kind {
+                record!(self.tables.super_predicates[def_id] <- self.tcx.super_predicates_of(def_id));
+            }
+        }
+        let inherent_impls = tcx.crate_inherent_impls(LOCAL_CRATE);
+        for (def_id, implementations) in inherent_impls.inherent_impls.iter() {
+            assert!(def_id.is_local());
+            if implementations.is_empty() {
+                continue;
+            }
+            record!(self.tables.inherent_impls[def_id] <- implementations.iter().map(|&def_id| {
+                assert!(def_id.is_local());
+                def_id.index
+            }));
         }
-    }
-
-    fn encode_variances_of(&mut self, def_id: DefId) {
-        debug!("EncodeContext::encode_variances_of({:?})", def_id);
-        record!(self.tables.variances[def_id] <- self.tcx.variances_of(def_id));
     }
 
     fn encode_item_type(&mut self, def_id: DefId) {
@@ -901,12 +988,7 @@ impl EncodeContext<'a, 'tcx> {
             if let Some(ctor_def_id) = variant.ctor_def_id {
                 record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(ctor_def_id));
             }
-            // FIXME(eddyb) is this ever used?
-            self.encode_variances_of(def_id);
         }
-        self.encode_generics(def_id);
-        self.encode_explicit_predicates(def_id);
-        self.encode_inferred_outlives(def_id);
     }
 
     fn encode_enum_variant_ctor(&mut self, def: &ty::AdtDef, index: VariantIdx) {
@@ -927,11 +1009,7 @@ impl EncodeContext<'a, 'tcx> {
         self.encode_item_type(def_id);
         if variant.ctor_kind == CtorKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            self.encode_variances_of(def_id);
         }
-        self.encode_generics(def_id);
-        self.encode_explicit_predicates(def_id);
-        self.encode_inferred_outlives(def_id);
     }
 
     fn encode_info_for_mod(&mut self, local_def_id: LocalDefId, md: &hir::Mod<'_>) {
@@ -990,9 +1068,6 @@ impl EncodeContext<'a, 'tcx> {
         record!(self.tables.kind[def_id] <- EntryKind::Field);
         self.encode_ident_span(def_id, field.ident);
         self.encode_item_type(def_id);
-        self.encode_generics(def_id);
-        self.encode_explicit_predicates(def_id);
-        self.encode_inferred_outlives(def_id);
     }
 
     fn encode_struct_ctor(&mut self, adt_def: &ty::AdtDef, def_id: DefId) {
@@ -1011,35 +1086,7 @@ impl EncodeContext<'a, 'tcx> {
         self.encode_item_type(def_id);
         if variant.ctor_kind == CtorKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            self.encode_variances_of(def_id);
         }
-        self.encode_generics(def_id);
-        self.encode_explicit_predicates(def_id);
-        self.encode_inferred_outlives(def_id);
-    }
-
-    fn encode_generics(&mut self, def_id: DefId) {
-        debug!("EncodeContext::encode_generics({:?})", def_id);
-        record!(self.tables.generics[def_id] <- self.tcx.generics_of(def_id));
-    }
-
-    fn encode_explicit_predicates(&mut self, def_id: DefId) {
-        debug!("EncodeContext::encode_explicit_predicates({:?})", def_id);
-        record!(self.tables.explicit_predicates[def_id] <-
-            self.tcx.explicit_predicates_of(def_id));
-    }
-
-    fn encode_inferred_outlives(&mut self, def_id: DefId) {
-        debug!("EncodeContext::encode_inferred_outlives({:?})", def_id);
-        let inferred_outlives = self.tcx.inferred_outlives_of(def_id);
-        if !inferred_outlives.is_empty() {
-            record!(self.tables.inferred_outlives[def_id] <- inferred_outlives);
-        }
-    }
-
-    fn encode_super_predicates(&mut self, def_id: DefId) {
-        debug!("EncodeContext::encode_super_predicates({:?})", def_id);
-        record!(self.tables.super_predicates[def_id] <- self.tcx.super_predicates_of(def_id));
     }
 
     fn encode_explicit_item_bounds(&mut self, def_id: DefId) {
@@ -1116,11 +1163,7 @@ impl EncodeContext<'a, 'tcx> {
         }
         if trait_item.kind == ty::AssocKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            self.encode_variances_of(def_id);
         }
-        self.encode_generics(def_id);
-        self.encode_explicit_predicates(def_id);
-        self.encode_inferred_outlives(def_id);
     }
 
     fn encode_info_for_impl_item(&mut self, def_id: DefId) {
@@ -1177,11 +1220,7 @@ impl EncodeContext<'a, 'tcx> {
         self.encode_item_type(def_id);
         if impl_item.kind == ty::AssocKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            self.encode_variances_of(def_id);
         }
-        self.encode_generics(def_id);
-        self.encode_explicit_predicates(def_id);
-        self.encode_inferred_outlives(def_id);
     }
 
     fn encode_fn_param_names_for_body(&mut self, body_id: hir::BodyId) -> Lazy<[Ident]> {
@@ -1236,18 +1275,6 @@ impl EncodeContext<'a, 'tcx> {
         }
     }
 
-    // Encodes the inherent implementations of a structure, enumeration, or trait.
-    fn encode_inherent_implementations(&mut self, def_id: DefId) {
-        debug!("EncodeContext::encode_inherent_implementations({:?})", def_id);
-        let implementations = self.tcx.inherent_impls(def_id);
-        if !implementations.is_empty() {
-            record!(self.tables.inherent_impls[def_id] <- implementations.iter().map(|&def_id| {
-                assert!(def_id.is_local());
-                def_id.index
-            }));
-        }
-    }
-
     fn encode_stability(&mut self, def_id: DefId) {
         debug!("EncodeContext::encode_stability({:?})", def_id);
 
@@ -1458,38 +1485,6 @@ impl EncodeContext<'a, 'tcx> {
                 record!(self.tables.impl_trait_ref[def_id] <- trait_ref);
             }
         }
-        self.encode_inherent_implementations(def_id);
-        match item.kind {
-            hir::ItemKind::Enum(..)
-            | hir::ItemKind::Struct(..)
-            | hir::ItemKind::Union(..)
-            | hir::ItemKind::Fn(..) => self.encode_variances_of(def_id),
-            _ => {}
-        }
-        match item.kind {
-            hir::ItemKind::Static(..)
-            | hir::ItemKind::Const(..)
-            | hir::ItemKind::Fn(..)
-            | hir::ItemKind::TyAlias(..)
-            | hir::ItemKind::Enum(..)
-            | hir::ItemKind::Struct(..)
-            | hir::ItemKind::Union(..)
-            | hir::ItemKind::Impl { .. }
-            | hir::ItemKind::OpaqueTy(..)
-            | hir::ItemKind::Trait(..)
-            | hir::ItemKind::TraitAlias(..) => {
-                self.encode_generics(def_id);
-                self.encode_explicit_predicates(def_id);
-                self.encode_inferred_outlives(def_id);
-            }
-            _ => {}
-        }
-        match item.kind {
-            hir::ItemKind::Trait(..) | hir::ItemKind::TraitAlias(..) => {
-                self.encode_super_predicates(def_id);
-            }
-            _ => {}
-        }
     }
 
     /// Serialize the text of exported macros
@@ -1530,7 +1525,6 @@ impl EncodeContext<'a, 'tcx> {
         if let ty::Closure(def_id, substs) = *ty.kind() {
             record!(self.tables.fn_sig[def_id] <- substs.as_closure().sig());
         }
-        self.encode_generics(def_id.to_def_id());
     }
 
     fn encode_info_for_anon_const(&mut self, def_id: LocalDefId) {
@@ -1542,9 +1536,6 @@ impl EncodeContext<'a, 'tcx> {
 
         record!(self.tables.kind[def_id.to_def_id()] <- EntryKind::AnonConst(qualifs, const_data));
         self.encode_item_type(def_id.to_def_id());
-        self.encode_generics(def_id.to_def_id());
-        self.encode_explicit_predicates(def_id.to_def_id());
-        self.encode_inferred_outlives(def_id.to_def_id());
     }
 
     fn encode_native_libraries(&mut self) -> Lazy<[NativeLib]> {
@@ -1821,14 +1812,9 @@ impl EncodeContext<'a, 'tcx> {
         }
         self.encode_ident_span(def_id, nitem.ident);
         self.encode_item_type(def_id);
-        self.encode_inherent_implementations(def_id);
         if let hir::ForeignItemKind::Fn(..) = nitem.kind {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            self.encode_variances_of(def_id);
         }
-        self.encode_generics(def_id);
-        self.encode_explicit_predicates(def_id);
-        self.encode_inferred_outlives(def_id);
     }
 }
 
@@ -1890,13 +1876,12 @@ impl EncodeContext<'a, 'tcx> {
                         default.is_some(),
                     );
                 }
-                GenericParamKind::Const { .. } => {
-                    self.encode_info_for_generic_param(
-                        def_id.to_def_id(),
-                        EntryKind::ConstParam,
-                        true,
-                    );
-                    // FIXME(const_generics_defaults)
+                GenericParamKind::Const { ref default, .. } => {
+                    let def_id = def_id.to_def_id();
+                    self.encode_info_for_generic_param(def_id, EntryKind::ConstParam, true);
+                    if default.is_some() {
+                        record!(self.tables.const_defaults[def_id] <- self.tcx.const_param_default(def_id))
+                    }
                 }
             }
         }
@@ -2072,10 +2057,10 @@ pub(super) fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata {
 
 fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
     let mut encoder = opaque::Encoder::new(vec![]);
-    encoder.emit_raw_bytes(METADATA_HEADER);
+    encoder.emit_raw_bytes(METADATA_HEADER).unwrap();
 
     // Will be filled with the root position after encoding everything.
-    encoder.emit_raw_bytes(&[0, 0, 0, 0]);
+    encoder.emit_raw_bytes(&[0, 0, 0, 0]).unwrap();
 
     let source_map_files = tcx.sess.source_map().files();
     let source_file_cache = (source_map_files[0].clone(), 0);
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index b44c3bfcac6..7cfb051e703 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -7,7 +7,7 @@ use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, DefKind};
-use rustc_hir::def_id::{DefId, DefIndex, DefPathHash};
+use rustc_hir::def_id::{DefId, DefIndex, DefPathHash, StableCrateId};
 use rustc_hir::definitions::DefKey;
 use rustc_hir::lang_items;
 use rustc_index::{bit_set::FiniteBitSet, vec::IndexVec};
@@ -203,6 +203,7 @@ crate struct CrateRoot<'tcx> {
     extra_filename: String,
     hash: Svh,
     disambiguator: CrateDisambiguator,
+    stable_crate_id: StableCrateId,
     panic_strategy: PanicStrategy,
     edition: Edition,
     has_global_allocator: bool,
@@ -306,13 +307,14 @@ define_tables! {
     mir_for_ctfe: Table<DefIndex, Lazy!(mir::Body<'tcx>)>,
     promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::Body<'tcx>>)>,
     mir_abstract_consts: Table<DefIndex, Lazy!(&'tcx [mir::abstract_const::Node<'tcx>])>,
+    const_defaults: Table<DefIndex, Lazy<rustc_middle::ty::Const<'tcx>>>,
     unused_generic_params: Table<DefIndex, Lazy<FiniteBitSet<u32>>>,
     // `def_keys` and `def_path_hashes` represent a lazy version of a
     // `DefPathTable`. This allows us to avoid deserializing an entire
     // `DefPathTable` up front, since we may only ever use a few
     // definitions from any given crate.
     def_keys: Table<DefIndex, Lazy<DefKey>>,
-    def_path_hashes: Table<DefIndex, Lazy<DefPathHash>>
+    def_path_hashes: Table<DefIndex, Lazy<DefPathHash>>,
 }
 
 #[derive(Copy, Clone, MetadataEncodable, MetadataDecodable)]
diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs
index 03bd4170ea9..62c0ce15845 100644
--- a/compiler/rustc_metadata/src/rmeta/table.rs
+++ b/compiler/rustc_metadata/src/rmeta/table.rs
@@ -2,6 +2,7 @@ use crate::rmeta::*;
 
 use rustc_index::vec::Idx;
 use rustc_serialize::opaque::Encoder;
+use rustc_serialize::Encoder as _;
 use std::convert::TryInto;
 use std::marker::PhantomData;
 use std::num::NonZeroUsize;
@@ -172,7 +173,7 @@ where
 
     pub(crate) fn encode(&self, buf: &mut Encoder) -> Lazy<Table<I, T>> {
         let pos = buf.position();
-        buf.emit_raw_bytes(&self.bytes);
+        buf.emit_raw_bytes(&self.bytes).unwrap();
         Lazy::from_position_and_meta(NonZeroUsize::new(pos as usize).unwrap(), self.bytes.len())
     }
 }
diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index 06742331655..8cb30e72f79 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -11,7 +11,7 @@ doctest = false
 rustc_arena = { path = "../rustc_arena" }
 bitflags = "1.2.1"
 tracing = "0.1"
-rustc-rayon-core = "0.3.0"
+rustc-rayon-core = "0.3.1"
 polonius-engine = "0.12.0"
 rustc_apfloat = { path = "../rustc_apfloat" }
 rustc_attr = { path = "../rustc_attr" }
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index ba9d0a40732..aa54d1ae7b9 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -32,8 +32,8 @@
 //! `DepNode` definition happens in the `define_dep_nodes!()` macro. This macro
 //! defines the `DepKind` enum. Each `DepKind` has its own parameters that are
 //! needed at runtime in order to construct a valid `DepNode` fingerprint.
-//! However, only `CompileCodegenUnit` is constructed explicitly (with
-//! `make_compile_codegen_unit`).
+//! However, only `CompileCodegenUnit` and `CompileMonoItem` are constructed
+//! explicitly (with `make_compile_codegen_unit` cq `make_compile_mono_item`).
 //!
 //! Because the macro sees what parameters a given `DepKind` requires, it can
 //! "infer" some properties for each kind of `DepNode`:
@@ -46,15 +46,17 @@
 //!   `DefId` it was computed from. In other cases, too much information gets
 //!   lost during fingerprint computation.
 //!
-//! `make_compile_codegen_unit`, together with `DepNode::new()`, ensures that only
-//! valid `DepNode` instances can be constructed. For example, the API does not
-//! allow for constructing parameterless `DepNode`s with anything other
-//! than a zeroed out fingerprint. More generally speaking, it relieves the
-//! user of the `DepNode` API of having to know how to compute the expected
-//! fingerprint for a given set of node parameters.
+//! `make_compile_codegen_unit` and `make_compile_mono_items`, together with
+//! `DepNode::new()`, ensures that only valid `DepNode` instances can be
+//! constructed. For example, the API does not allow for constructing
+//! parameterless `DepNode`s with anything other than a zeroed out fingerprint.
+//! More generally speaking, it relieves the user of the `DepNode` API of
+//! having to know how to compute the expected fingerprint for a given set of
+//! node parameters.
 //!
 //! [dependency graph]: https://rustc-dev-guide.rust-lang.org/query.html
 
+use crate::mir::mono::MonoItem;
 use crate::ty::TyCtxt;
 
 use rustc_data_structures::fingerprint::Fingerprint;
@@ -175,6 +177,14 @@ pub mod dep_kind {
         can_reconstruct_query_key: || false,
     };
 
+    pub const CompileMonoItem: DepKindStruct = DepKindStruct {
+        has_params: true,
+        is_anon: false,
+        is_eval_always: false,
+
+        can_reconstruct_query_key: || false,
+    };
+
     macro_rules! define_query_dep_kinds {
         ($(
             [$($attrs:tt)*]
@@ -251,6 +261,10 @@ rustc_dep_node_append!([define_dep_nodes!][ <'tcx>
 
     // WARNING: if `Symbol` is changed, make sure you update `make_compile_codegen_unit` below.
     [] CompileCodegenUnit(Symbol),
+
+    // WARNING: if `MonoItem` is changed, make sure you update `make_compile_mono_item` below.
+    // Only used by rustc_codegen_cranelift
+    [] CompileMonoItem(MonoItem),
 ]);
 
 // WARNING: `construct` is generic and does not know that `CompileCodegenUnit` takes `Symbol`s as keys.
@@ -259,6 +273,12 @@ crate fn make_compile_codegen_unit(tcx: TyCtxt<'_>, name: Symbol) -> DepNode {
     DepNode::construct(tcx, DepKind::CompileCodegenUnit, &name)
 }
 
+// WARNING: `construct` is generic and does not know that `CompileMonoItem` takes `MonoItem`s as keys.
+// Be very careful changing this type signature!
+crate fn make_compile_mono_item(tcx: TyCtxt<'tcx>, mono_item: &MonoItem<'tcx>) -> DepNode {
+    DepNode::construct(tcx, DepKind::CompileMonoItem, mono_item)
+}
+
 pub type DepNode = rustc_query_system::dep_graph::DepNode<DepKind>;
 
 // We keep a lot of `DepNode`s in memory during compilation. It's not
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index c688b23be1d..31bea832958 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -8,18 +8,19 @@ use rustc_session::Session;
 mod dep_node;
 
 pub use rustc_query_system::dep_graph::{
-    debug, hash_result, DepContext, DepNodeColor, DepNodeIndex, SerializedDepNodeIndex,
-    WorkProduct, WorkProductId,
+    debug::DepNodeFilter, hash_result, DepContext, DepNodeColor, DepNodeIndex,
+    SerializedDepNodeIndex, WorkProduct, WorkProductId,
 };
 
-crate use dep_node::make_compile_codegen_unit;
 pub use dep_node::{label_strs, DepKind, DepNode, DepNodeExt};
+crate use dep_node::{make_compile_codegen_unit, make_compile_mono_item};
 
 pub type DepGraph = rustc_query_system::dep_graph::DepGraph<DepKind>;
 pub type TaskDeps = rustc_query_system::dep_graph::TaskDeps<DepKind>;
 pub type DepGraphQuery = rustc_query_system::dep_graph::DepGraphQuery<DepKind>;
 pub type PreviousDepGraph = rustc_query_system::dep_graph::PreviousDepGraph<DepKind>;
 pub type SerializedDepGraph = rustc_query_system::dep_graph::SerializedDepGraph<DepKind>;
+pub type EdgeFilter = rustc_query_system::dep_graph::debug::EdgeFilter<DepKind>;
 
 impl rustc_query_system::dep_graph::DepKind for DepKind {
     const NULL: Self = DepKind::Null;
diff --git a/compiler/rustc_middle/src/hir/map/blocks.rs b/compiler/rustc_middle/src/hir/map/blocks.rs
index 9222ce1015e..706c7900949 100644
--- a/compiler/rustc_middle/src/hir/map/blocks.rs
+++ b/compiler/rustc_middle/src/hir/map/blocks.rs
@@ -12,7 +12,6 @@
 //! for the `Code` associated with a particular NodeId.
 
 use crate::hir::map::Map;
-use rustc_ast::Attribute;
 use rustc_hir as hir;
 use rustc_hir::intravisit::FnKind;
 use rustc_hir::{Expr, FnDecl, Node};
@@ -105,7 +104,6 @@ struct ItemFnParts<'a> {
     body: hir::BodyId,
     id: hir::HirId,
     span: Span,
-    attrs: &'a [Attribute],
 }
 
 /// These are all the components one can extract from a closure expr
@@ -115,18 +113,11 @@ struct ClosureParts<'a> {
     body: hir::BodyId,
     id: hir::HirId,
     span: Span,
-    attrs: &'a [Attribute],
 }
 
 impl<'a> ClosureParts<'a> {
-    fn new(
-        d: &'a FnDecl<'a>,
-        b: hir::BodyId,
-        id: hir::HirId,
-        s: Span,
-        attrs: &'a [Attribute],
-    ) -> Self {
-        ClosureParts { decl: d, body: b, id, span: s, attrs }
+    fn new(d: &'a FnDecl<'a>, b: hir::BodyId, id: hir::HirId, s: Span) -> Self {
+        ClosureParts { decl: d, body: b, id, span: s }
     }
 }
 
@@ -146,7 +137,7 @@ impl<'a> FnLikeNode<'a> {
     pub fn body(self) -> hir::BodyId {
         self.handle(
             |i: ItemFnParts<'a>| i.body,
-            |_, _, _: &'a hir::FnSig<'a>, _, body: hir::BodyId, _, _| body,
+            |_, _, _: &'a hir::FnSig<'a>, _, body: hir::BodyId, _| body,
             |c: ClosureParts<'a>| c.body,
         )
     }
@@ -154,7 +145,7 @@ impl<'a> FnLikeNode<'a> {
     pub fn decl(self) -> &'a FnDecl<'a> {
         self.handle(
             |i: ItemFnParts<'a>| &*i.decl,
-            |_, _, sig: &'a hir::FnSig<'a>, _, _, _, _| &sig.decl,
+            |_, _, sig: &'a hir::FnSig<'a>, _, _, _| &sig.decl,
             |c: ClosureParts<'a>| c.decl,
         )
     }
@@ -162,7 +153,7 @@ impl<'a> FnLikeNode<'a> {
     pub fn span(self) -> Span {
         self.handle(
             |i: ItemFnParts<'_>| i.span,
-            |_, _, _: &'a hir::FnSig<'a>, _, _, span, _| span,
+            |_, _, _: &'a hir::FnSig<'a>, _, _, span| span,
             |c: ClosureParts<'_>| c.span,
         )
     }
@@ -170,7 +161,7 @@ impl<'a> FnLikeNode<'a> {
     pub fn id(self) -> hir::HirId {
         self.handle(
             |i: ItemFnParts<'_>| i.id,
-            |id, _, _: &'a hir::FnSig<'a>, _, _, _, _| id,
+            |id, _, _: &'a hir::FnSig<'a>, _, _, _| id,
             |c: ClosureParts<'_>| c.id,
         )
     }
@@ -189,12 +180,11 @@ impl<'a> FnLikeNode<'a> {
 
     pub fn kind(self) -> FnKind<'a> {
         let item = |p: ItemFnParts<'a>| -> FnKind<'a> {
-            FnKind::ItemFn(p.ident, p.generics, p.header, p.vis, p.attrs)
-        };
-        let closure = |c: ClosureParts<'a>| FnKind::Closure(c.attrs);
-        let method = |_, ident: Ident, sig: &'a hir::FnSig<'a>, vis, _, _, attrs| {
-            FnKind::Method(ident, sig, vis, attrs)
+            FnKind::ItemFn(p.ident, p.generics, p.header, p.vis)
         };
+        let closure = |_: ClosureParts<'a>| FnKind::Closure;
+        let method =
+            |_, ident: Ident, sig: &'a hir::FnSig<'a>, vis, _, _| FnKind::Method(ident, sig, vis);
         self.handle(item, method, closure)
     }
 
@@ -208,7 +198,6 @@ impl<'a> FnLikeNode<'a> {
             Option<&'a hir::Visibility<'a>>,
             hir::BodyId,
             Span,
-            &'a [Attribute],
         ) -> A,
         C: FnOnce(ClosureParts<'a>) -> A,
     {
@@ -221,7 +210,6 @@ impl<'a> FnLikeNode<'a> {
                     body: block,
                     vis: &i.vis,
                     span: i.span,
-                    attrs: &i.attrs,
                     header: sig.header,
                     generics,
                 }),
@@ -229,19 +217,19 @@ impl<'a> FnLikeNode<'a> {
             },
             Node::TraitItem(ti) => match ti.kind {
                 hir::TraitItemKind::Fn(ref sig, hir::TraitFn::Provided(body)) => {
-                    method(ti.hir_id(), ti.ident, sig, None, body, ti.span, &ti.attrs)
+                    method(ti.hir_id(), ti.ident, sig, None, body, ti.span)
                 }
                 _ => bug!("trait method FnLikeNode that is not fn-like"),
             },
             Node::ImplItem(ii) => match ii.kind {
                 hir::ImplItemKind::Fn(ref sig, body) => {
-                    method(ii.hir_id(), ii.ident, sig, Some(&ii.vis), body, ii.span, &ii.attrs)
+                    method(ii.hir_id(), ii.ident, sig, Some(&ii.vis), body, ii.span)
                 }
                 _ => bug!("impl method FnLikeNode that is not fn-like"),
             },
             Node::Expr(e) => match e.kind {
                 hir::ExprKind::Closure(_, ref decl, block, _fn_decl_span, _gen) => {
-                    closure(ClosureParts::new(&decl, block, e.hir_id, e.span, &e.attrs))
+                    closure(ClosureParts::new(&decl, block, e.hir_id, e.span))
                 }
                 _ => bug!("expr FnLikeNode that is not fn-like"),
             },
diff --git a/compiler/rustc_middle/src/hir/map/collector.rs b/compiler/rustc_middle/src/hir/map/collector.rs
index a5ffa9c7a54..501e7d624d2 100644
--- a/compiler/rustc_middle/src/hir/map/collector.rs
+++ b/compiler/rustc_middle/src/hir/map/collector.rs
@@ -52,6 +52,7 @@ fn insert_vec_map<K: Idx, V: Clone>(map: &mut IndexVec<K, Option<V>>, k: K, v: V
     if i >= len {
         map.extend(repeat(None).take(i - len + 1));
     }
+    debug_assert!(map[k].is_none());
     map[k] = Some(v);
 }
 
@@ -116,6 +117,7 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> {
                 modules: _,
                 proc_macros: _,
                 trait_map: _,
+                attrs: _,
             } = *krate;
 
             hash_body(&mut hcx, root_mod_def_path_hash, item, &mut hir_body_nodes)
@@ -215,11 +217,21 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> {
             // Overwrite the dummy hash with the real HIR owner hash.
             nodes.hash = hash;
 
-            // FIXME: feature(impl_trait_in_bindings) broken and trigger this assert
-            //assert!(data.signature.is_none());
-
+            debug_assert!(data.signature.is_none());
             data.signature =
                 Some(self.arena.alloc(Owner { parent: entry.parent, node: entry.node }));
+
+            let dk_parent = self.definitions.def_key(id.owner).parent;
+            if let Some(dk_parent) = dk_parent {
+                let dk_parent = LocalDefId { local_def_index: dk_parent };
+                let dk_parent = self.definitions.local_def_id_to_hir_id(dk_parent);
+                if dk_parent.owner != entry.parent.owner {
+                    panic!(
+                        "Different parents for {:?} => dk_parent={:?} actual={:?}",
+                        id.owner, dk_parent, entry.parent,
+                    )
+                }
+            }
         } else {
             assert_eq!(entry.parent.owner, id.owner);
             insert_vec_map(
@@ -361,26 +373,12 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> {
     }
 
     fn visit_generic_param(&mut self, param: &'hir GenericParam<'hir>) {
-        if let hir::GenericParamKind::Type {
-            synthetic: Some(hir::SyntheticTyParamKind::ImplTrait),
-            ..
-        } = param.kind
-        {
-            debug_assert_eq!(
-                param.hir_id.owner,
-                self.definitions.opt_hir_id_to_local_def_id(param.hir_id).unwrap()
-            );
-            self.with_dep_node_owner(param.hir_id.owner, param, |this, hash| {
-                this.insert_with_hash(param.span, param.hir_id, Node::GenericParam(param), hash);
+        self.insert(param.span, param.hir_id, Node::GenericParam(param));
+        intravisit::walk_generic_param(self, param);
+    }
 
-                this.with_parent(param.hir_id, |this| {
-                    intravisit::walk_generic_param(this, param);
-                });
-            });
-        } else {
-            self.insert(param.span, param.hir_id, Node::GenericParam(param));
-            intravisit::walk_generic_param(self, param);
-        }
+    fn visit_const_param_default(&mut self, param: HirId, ct: &'hir AnonConst) {
+        self.with_parent(param, |this| intravisit::walk_const_param_default(this, ct))
     }
 
     fn visit_trait_item(&mut self, ti: &'hir TraitItem<'hir>) {
@@ -541,10 +539,10 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> {
         });
     }
 
-    fn visit_struct_field(&mut self, field: &'hir StructField<'hir>) {
+    fn visit_field_def(&mut self, field: &'hir FieldDef<'hir>) {
         self.insert(field.span, field.hir_id, Node::Field(field));
         self.with_parent(field.hir_id, |this| {
-            intravisit::walk_struct_field(this, field);
+            intravisit::walk_field_def(this, field);
         });
     }
 
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index 5c2bd575e7d..d155276051e 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -180,11 +180,6 @@ impl<'hir> Map<'hir> {
         self.tcx.definitions.local_def_id_to_hir_id(def_id)
     }
 
-    #[inline]
-    pub fn opt_local_def_id_to_hir_id(&self, def_id: LocalDefId) -> Option<HirId> {
-        self.tcx.definitions.opt_local_def_id_to_hir_id(def_id)
-    }
-
     pub fn iter_local_def_id(&self) -> impl Iterator<Item = LocalDefId> + '_ {
         self.tcx.definitions.iter_local_def_id()
     }
@@ -457,17 +452,14 @@ impl<'hir> Map<'hir> {
     /// invoking `krate.attrs` because it registers a tighter
     /// dep-graph access.
     pub fn krate_attrs(&self) -> &'hir [ast::Attribute] {
-        match self.get_entry(CRATE_HIR_ID).node {
-            Node::Crate(item) => item.attrs,
-            _ => bug!(),
-        }
+        self.attrs(CRATE_HIR_ID)
     }
 
     pub fn get_module(&self, module: LocalDefId) -> (&'hir Mod<'hir>, Span, HirId) {
         let hir_id = self.local_def_id_to_hir_id(module);
         match self.get_entry(hir_id).node {
             Node::Item(&Item { span, kind: ItemKind::Mod(ref m), .. }) => (m, span, hir_id),
-            Node::Crate(item) => (&item.module, item.span, hir_id),
+            Node::Crate(item) => (&item, item.inner, hir_id),
             node => panic!("not a module: {:?}", node),
         }
     }
@@ -558,24 +550,6 @@ impl<'hir> Map<'hir> {
         ParentHirIterator { current_id, map: self }
     }
 
-    /// Checks if the node is an argument. An argument is a local variable whose
-    /// immediate parent is an item or a closure.
-    pub fn is_argument(&self, id: HirId) -> bool {
-        match self.find(id) {
-            Some(Node::Binding(_)) => (),
-            _ => return false,
-        }
-        matches!(
-            self.find(self.get_parent_node(id)),
-            Some(
-                Node::Item(_)
-                    | Node::TraitItem(_)
-                    | Node::ImplItem(_)
-                    | Node::Expr(Expr { kind: ExprKind::Closure(..), .. }),
-            )
-        )
-    }
-
     /// Checks if the node is left-hand side of an assignment.
     pub fn is_lhs(&self, id: HirId) -> bool {
         match self.find(self.get_parent_node(id)) {
@@ -787,17 +761,6 @@ impl<'hir> Map<'hir> {
         }
     }
 
-    pub fn expect_variant_data(&self, id: HirId) -> &'hir VariantData<'hir> {
-        match self.find(id) {
-            Some(
-                Node::Ctor(vd)
-                | Node::Item(Item { kind: ItemKind::Struct(vd, _) | ItemKind::Union(vd, _), .. }),
-            ) => vd,
-            Some(Node::Variant(variant)) => &variant.data,
-            _ => bug!("expected struct or variant, found {}", self.node_to_string(id)),
-        }
-    }
-
     pub fn expect_variant(&self, id: HirId) -> &'hir Variant<'hir> {
         match self.find(id) {
             Some(Node::Variant(variant)) => variant,
@@ -853,34 +816,7 @@ impl<'hir> Map<'hir> {
     /// Given a node ID, gets a list of attributes associated with the AST
     /// corresponding to the node-ID.
     pub fn attrs(&self, id: HirId) -> &'hir [ast::Attribute] {
-        self.find_entry(id).map_or(&[], |entry| match entry.node {
-            Node::Param(a) => a.attrs,
-            Node::Local(l) => &l.attrs[..],
-            Node::Item(i) => i.attrs,
-            Node::ForeignItem(fi) => fi.attrs,
-            Node::TraitItem(ref ti) => ti.attrs,
-            Node::ImplItem(ref ii) => ii.attrs,
-            Node::Variant(ref v) => v.attrs,
-            Node::Field(ref f) => f.attrs,
-            Node::Expr(ref e) => &*e.attrs,
-            Node::Stmt(ref s) => s.kind.attrs(|id| self.item(id)),
-            Node::Arm(ref a) => &*a.attrs,
-            Node::GenericParam(param) => param.attrs,
-            // Unit/tuple structs/variants take the attributes straight from
-            // the struct/variant definition.
-            Node::Ctor(..) => self.attrs(self.get_parent_item(id)),
-            Node::Crate(item) => item.attrs,
-            Node::MacroDef(def) => def.attrs,
-            Node::AnonConst(..)
-            | Node::PathSegment(..)
-            | Node::Ty(..)
-            | Node::Pat(..)
-            | Node::Binding(..)
-            | Node::TraitRef(..)
-            | Node::Block(..)
-            | Node::Lifetime(..)
-            | Node::Visibility(..) => &[],
-        })
+        self.tcx.hir_attrs(id.owner).get(id.local_id)
     }
 
     /// Gets the span of the definition of the specified HIR node.
@@ -932,7 +868,7 @@ impl<'hir> Map<'hir> {
             Node::Visibility(v) => bug!("unexpected Visibility {:?}", v),
             Node::Local(local) => local.span,
             Node::MacroDef(macro_def) => macro_def.span,
-            Node::Crate(item) => item.span,
+            Node::Crate(item) => item.inner,
         };
         Some(span)
     }
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index 5f9cf8771ea..cf4e473d8ac 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -9,6 +9,7 @@ pub mod place;
 use crate::ich::StableHashingContext;
 use crate::ty::query::Providers;
 use crate::ty::TyCtxt;
+use rustc_ast::Attribute;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -16,6 +17,7 @@ use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE};
 use rustc_hir::*;
 use rustc_index::vec::IndexVec;
 use rustc_span::DUMMY_SP;
+use std::collections::BTreeMap;
 
 #[derive(Debug)]
 pub struct Owner<'tcx> {
@@ -55,6 +57,48 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for OwnerNodes<'tcx> {
     }
 }
 
+#[derive(Copy, Clone)]
+pub struct AttributeMap<'tcx> {
+    map: &'tcx BTreeMap<HirId, &'tcx [Attribute]>,
+    prefix: LocalDefId,
+}
+
+impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for AttributeMap<'tcx> {
+    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
+        let range = self.range();
+
+        range.clone().count().hash_stable(hcx, hasher);
+        for (key, value) in range {
+            key.hash_stable(hcx, hasher);
+            value.hash_stable(hcx, hasher);
+        }
+    }
+}
+
+impl<'tcx> std::fmt::Debug for AttributeMap<'tcx> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("AttributeMap")
+            .field("prefix", &self.prefix)
+            .field("range", &&self.range().collect::<Vec<_>>()[..])
+            .finish()
+    }
+}
+
+impl<'tcx> AttributeMap<'tcx> {
+    fn get(&self, id: ItemLocalId) -> &'tcx [Attribute] {
+        self.map.get(&HirId { owner: self.prefix, local_id: id }).copied().unwrap_or(&[])
+    }
+
+    fn range(&self) -> std::collections::btree_map::Range<'_, rustc_hir::HirId, &[Attribute]> {
+        let local_zero = ItemLocalId::from_u32(0);
+        let range = HirId { owner: self.prefix, local_id: local_zero }..HirId {
+            owner: LocalDefId { local_def_index: self.prefix.local_def_index + 1 },
+            local_id: local_zero,
+        };
+        self.map.range(range)
+    }
+}
+
 impl<'tcx> TyCtxt<'tcx> {
     #[inline(always)]
     pub fn hir(self) -> map::Map<'tcx> {
@@ -76,6 +120,7 @@ pub fn provide(providers: &mut Providers) {
     providers.hir_module_items = |tcx, id| &tcx.untracked_crate.modules[&id];
     providers.hir_owner = |tcx, id| tcx.index_hir(LOCAL_CRATE).map[id].signature;
     providers.hir_owner_nodes = |tcx, id| tcx.index_hir(LOCAL_CRATE).map[id].with_bodies.as_deref();
+    providers.hir_attrs = |tcx, id| AttributeMap { map: &tcx.untracked_crate.attrs, prefix: id };
     providers.def_span = |tcx, def_id| tcx.hir().span_if_local(def_id).unwrap_or(DUMMY_SP);
     providers.fn_arg_names = |tcx, id| {
         let hir = tcx.hir();
diff --git a/compiler/rustc_middle/src/ich/hcx.rs b/compiler/rustc_middle/src/ich/hcx.rs
index 51b650e5ade..b2fef731b7e 100644
--- a/compiler/rustc_middle/src/ich/hcx.rs
+++ b/compiler/rustc_middle/src/ich/hcx.rs
@@ -120,11 +120,6 @@ impl<'a> StableHashingContext<'a> {
     }
 
     #[inline]
-    pub fn sess(&self) -> &'a Session {
-        self.sess
-    }
-
-    #[inline]
     pub fn while_hashing_hir_bodies<F: FnOnce(&mut Self)>(&mut self, hash_bodies: bool, f: F) {
         let prev_hash_bodies = self.hash_bodies;
         self.hash_bodies = hash_bodies;
@@ -250,13 +245,6 @@ impl<'a> rustc_span::HashStableContext for StableHashingContext<'a> {
         &CACHE
     }
 
-    fn byte_pos_to_line_and_col(
-        &mut self,
-        byte: BytePos,
-    ) -> Option<(Lrc<SourceFile>, usize, BytePos)> {
-        self.source_map().byte_pos_to_line_and_col(byte)
-    }
-
     fn span_data_to_lines_and_cols(
         &mut self,
         span: &SpanData,
diff --git a/compiler/rustc_middle/src/ich/impls_hir.rs b/compiler/rustc_middle/src/ich/impls_hir.rs
index 5ef70a89051..abf56832329 100644
--- a/compiler/rustc_middle/src/ich/impls_hir.rs
+++ b/compiler/rustc_middle/src/ich/impls_hir.rs
@@ -66,11 +66,10 @@ impl<'ctx> rustc_hir::HashStableContext for StableHashingContext<'ctx> {
 
     fn hash_hir_expr(&mut self, expr: &hir::Expr<'_>, hasher: &mut StableHasher) {
         self.while_hashing_hir_bodies(true, |hcx| {
-            let hir::Expr { hir_id: _, ref span, ref kind, ref attrs } = *expr;
+            let hir::Expr { hir_id: _, ref span, ref kind } = *expr;
 
             span.hash_stable(hcx, hasher);
             kind.hash_stable(hcx, hasher);
-            attrs.hash_stable(hcx, hasher);
         })
     }
 
diff --git a/compiler/rustc_middle/src/ich/impls_syntax.rs b/compiler/rustc_middle/src/ich/impls_syntax.rs
index bfbe15749ee..b93b25d6b5c 100644
--- a/compiler/rustc_middle/src/ich/impls_syntax.rs
+++ b/compiler/rustc_middle/src/ich/impls_syntax.rs
@@ -1,5 +1,5 @@
 //! This module contains `HashStable` implementations for various data types
-//! from librustc_ast in no particular order.
+//! from `rustc_ast` in no particular order.
 
 use crate::ich::StableHashingContext;
 
@@ -45,7 +45,11 @@ impl<'ctx> rustc_ast::HashStableContext for StableHashingContext<'ctx> {
             item.hash_stable(self, hasher);
             style.hash_stable(self, hasher);
             span.hash_stable(self, hasher);
-            tokens.as_ref().expect_none("Tokens should have been removed during lowering!");
+            assert_matches!(
+                tokens.as_ref(),
+                None,
+                "Tokens should have been removed during lowering!"
+            );
         } else {
             unreachable!();
         }
diff --git a/compiler/rustc_middle/src/ich/impls_ty.rs b/compiler/rustc_middle/src/ich/impls_ty.rs
index 573b514e844..8e53e4ba948 100644
--- a/compiler/rustc_middle/src/ich/impls_ty.rs
+++ b/compiler/rustc_middle/src/ich/impls_ty.rs
@@ -70,16 +70,16 @@ impl<'a> HashStable<StableHashingContext<'a>> for ty::RegionKind {
             ty::ReEmpty(universe) => {
                 universe.hash_stable(hcx, hasher);
             }
-            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrAnon(i) }) => {
+            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrAnon(i), .. }) => {
                 db.hash_stable(hcx, hasher);
                 i.hash_stable(hcx, hasher);
             }
-            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrNamed(def_id, name) }) => {
+            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrNamed(def_id, name), .. }) => {
                 db.hash_stable(hcx, hasher);
                 def_id.hash_stable(hcx, hasher);
                 name.hash_stable(hcx, hasher);
             }
-            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrEnv }) => {
+            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrEnv, .. }) => {
                 db.hash_stable(hcx, hasher);
             }
             ty::ReEarlyBound(ty::EarlyBoundRegion { def_id, index, name }) => {
@@ -118,12 +118,13 @@ impl<'tcx> HashStable<StableHashingContext<'tcx>> for ty::BoundVar {
     }
 }
 
-impl<'a, T> HashStable<StableHashingContext<'a>> for ty::Binder<T>
+impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for ty::Binder<'tcx, T>
 where
     T: HashStable<StableHashingContext<'a>>,
 {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
         self.as_ref().skip_binder().hash_stable(hcx, hasher);
+        self.bound_vars().hash_stable(hcx, hasher);
     }
 }
 
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index e106db38b2c..5df2f91f09f 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -27,6 +27,7 @@ use crate::ty::{self, BoundVar, List, Region, TyCtxt};
 use rustc_index::vec::IndexVec;
 use rustc_macros::HashStable;
 use smallvec::SmallVec;
+use std::iter;
 use std::ops::Index;
 
 /// A "canonicalized" type `V` is one where all free inference
@@ -227,20 +228,12 @@ impl Certainty {
             Certainty::Ambiguous => false,
         }
     }
-
-    pub fn is_ambiguous(&self) -> bool {
-        !self.is_proven()
-    }
 }
 
 impl<'tcx, R> QueryResponse<'tcx, R> {
     pub fn is_proven(&self) -> bool {
         self.certainty.is_proven()
     }
-
-    pub fn is_ambiguous(&self) -> bool {
-        !self.is_proven()
-    }
 }
 
 impl<'tcx, R> Canonical<'tcx, QueryResponse<'tcx, R>> {
@@ -284,7 +277,7 @@ impl<'tcx, V> Canonical<'tcx, V> {
 }
 
 pub type QueryOutlivesConstraint<'tcx> =
-    ty::Binder<ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>>;
+    ty::Binder<'tcx, ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>>;
 
 TrivialTypeFoldableAndLiftImpls! {
     for <'tcx> {
@@ -315,16 +308,14 @@ impl<'tcx> CanonicalVarValues<'tcx> {
         use crate::ty::subst::GenericArgKind;
 
         CanonicalVarValues {
-            var_values: self
-                .var_values
-                .iter()
-                .zip(0..)
+            var_values: iter::zip(&self.var_values, 0..)
                 .map(|(kind, i)| match kind.unpack() {
                     GenericArgKind::Type(..) => {
                         tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_u32(i).into())).into()
                     }
                     GenericArgKind::Lifetime(..) => {
-                        let br = ty::BoundRegion { kind: ty::BrAnon(i) };
+                        let br =
+                            ty::BoundRegion { var: ty::BoundVar::from_u32(i), kind: ty::BrAnon(i) };
                         tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into()
                     }
                     GenericArgKind::Const(ct) => tcx
diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs
index 8318bdefc8e..641cf23781e 100644
--- a/compiler/rustc_middle/src/infer/unify_key.rs
+++ b/compiler/rustc_middle/src/infer/unify_key.rs
@@ -97,13 +97,6 @@ impl<'tcx> ConstVariableValue<'tcx> {
             ConstVariableValue::Known { value } => Some(value),
         }
     }
-
-    pub fn is_unknown(&self) -> bool {
-        match *self {
-            ConstVariableValue::Unknown { .. } => true,
-            ConstVariableValue::Known { .. } => false,
-        }
-    }
 }
 
 #[derive(Copy, Clone, Debug)]
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index d9e88265050..1db03e9165b 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -24,12 +24,11 @@
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(array_windows)]
-#![feature(assoc_char_funcs)]
+#![feature(assert_matches)]
 #![feature(backtrace)]
 #![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
-#![feature(cmp_min_max_by)]
 #![feature(const_fn)]
 #![feature(const_panic)]
 #![feature(core_intrinsics)]
@@ -38,8 +37,7 @@
 #![feature(extern_types)]
 #![feature(nll)]
 #![feature(once_cell)]
-#![feature(option_expect_none)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(min_specialization)]
 #![feature(trusted_len)]
 #![feature(test)]
@@ -52,6 +50,7 @@
 #![feature(exclusive_range_pattern)]
 #![feature(control_flow_enum)]
 #![feature(associated_type_defaults)]
+#![feature(iter_zip)]
 #![recursion_limit = "512"]
 
 #[macro_use]
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index 5f2ffda642c..7024d9a3d21 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -1,7 +1,7 @@
 use crate::mir::mono::Linkage;
 use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr};
-use rustc_session::config::SanitizerSet;
 use rustc_span::symbol::Symbol;
+use rustc_target::spec::SanitizerSet;
 
 #[derive(Clone, TyEncodable, TyDecodable, HashStable, Debug)]
 pub struct CodegenFnAttrs {
@@ -38,6 +38,9 @@ pub struct CodegenFnAttrs {
     /// be generated against a specific instruction set. Only usable on architectures which allow
     /// switching between multiple instruction sets.
     pub instruction_set: Option<InstructionSetAttr>,
+    /// The `#[repr(align(...))]` attribute. Indicates the value of which the function should be
+    /// aligned to.
+    pub alignment: Option<u32>,
 }
 
 bitflags! {
@@ -103,6 +106,7 @@ impl CodegenFnAttrs {
             link_section: None,
             no_sanitize: SanitizerSet::empty(),
             instruction_set: None,
+            alignment: None,
         }
     }
 
diff --git a/compiler/rustc_middle/src/middle/region.rs b/compiler/rustc_middle/src/middle/region.rs
index eb48198991c..f44267a404b 100644
--- a/compiler/rustc_middle/src/middle/region.rs
+++ b/compiler/rustc_middle/src/middle/region.rs
@@ -235,18 +235,6 @@ pub struct ScopeTree {
     /// escape into 'static and should have no local cleanup scope.
     rvalue_scopes: FxHashMap<hir::ItemLocalId, Option<Scope>>,
 
-    /// Encodes the hierarchy of fn bodies. Every fn body (including
-    /// closures) forms its own distinct region hierarchy, rooted in
-    /// the block that is the fn body. This map points from the ID of
-    /// that root block to the ID of the root block for the enclosing
-    /// fn, if any. Thus the map structures the fn bodies into a
-    /// hierarchy based on their lexical mapping. This is used to
-    /// handle the relationships between regions in a fn and in a
-    /// closure defined by that fn. See the "Modeling closures"
-    /// section of the README in infer::region_constraints for
-    /// more details.
-    closure_tree: FxHashMap<hir::ItemLocalId, hir::ItemLocalId>,
-
     /// If there are any `yield` nested within a scope, this map
     /// stores the `Span` of the last one and its index in the
     /// postorder of the Visitor traversal on the HIR.
@@ -356,23 +344,6 @@ impl ScopeTree {
         self.destruction_scopes.get(&n).cloned()
     }
 
-    /// Records that `sub_closure` is defined within `sup_closure`. These IDs
-    /// should be the ID of the block that is the fn body, which is
-    /// also the root of the region hierarchy for that fn.
-    pub fn record_closure_parent(
-        &mut self,
-        sub_closure: hir::ItemLocalId,
-        sup_closure: hir::ItemLocalId,
-    ) {
-        debug!(
-            "record_closure_parent(sub_closure={:?}, sup_closure={:?})",
-            sub_closure, sup_closure
-        );
-        assert!(sub_closure != sup_closure);
-        let previous = self.closure_tree.insert(sub_closure, sup_closure);
-        assert!(previous.is_none());
-    }
-
     pub fn record_var_scope(&mut self, var: hir::ItemLocalId, lifetime: Scope) {
         debug!("record_var_scope(sub={:?}, sup={:?})", var, lifetime);
         assert!(var != lifetime.item_local_id());
@@ -430,6 +401,8 @@ impl ScopeTree {
 
     /// Returns `true` if `subscope` is equal to or is lexically nested inside `superscope`, and
     /// `false` otherwise.
+    ///
+    /// Used by clippy.
     pub fn is_subscope_of(&self, subscope: Scope, superscope: Scope) -> bool {
         let mut s = subscope;
         debug!("is_subscope_of({:?}, {:?})", subscope, superscope);
@@ -472,7 +445,6 @@ impl<'a> HashStable<StableHashingContext<'a>> for ScopeTree {
             ref var_map,
             ref destruction_scopes,
             ref rvalue_scopes,
-            ref closure_tree,
             ref yield_in_scope,
         } = *self;
 
@@ -486,7 +458,6 @@ impl<'a> HashStable<StableHashingContext<'a>> for ScopeTree {
         var_map.hash_stable(hcx, hasher);
         destruction_scopes.hash_stable(hcx, hasher);
         rvalue_scopes.hash_stable(hcx, hasher);
-        closure_tree.hash_stable(hcx, hasher);
         yield_in_scope.hash_stable(hcx, hasher);
     }
 }
diff --git a/compiler/rustc_middle/src/middle/resolve_lifetime.rs b/compiler/rustc_middle/src/middle/resolve_lifetime.rs
index 1b7d0e620a4..aa6488b329e 100644
--- a/compiler/rustc_middle/src/middle/resolve_lifetime.rs
+++ b/compiler/rustc_middle/src/middle/resolve_lifetime.rs
@@ -39,8 +39,13 @@ impl LifetimeDefOrigin {
 pub enum Region {
     Static,
     EarlyBound(/* index */ u32, /* lifetime decl */ DefId, LifetimeDefOrigin),
-    LateBound(ty::DebruijnIndex, /* lifetime decl */ DefId, LifetimeDefOrigin),
-    LateBoundAnon(ty::DebruijnIndex, /* anon index */ u32),
+    LateBound(
+        ty::DebruijnIndex,
+        /* late-bound index */ u32,
+        /* lifetime decl */ DefId,
+        LifetimeDefOrigin,
+    ),
+    LateBoundAnon(ty::DebruijnIndex, /* late-bound index */ u32, /* anon index */ u32),
     Free(DefId, /* lifetime decl */ DefId),
 }
 
@@ -79,8 +84,5 @@ pub struct ResolveLifetimes {
     /// (b) it DOES appear in the arguments.
     pub late_bound: FxHashMap<LocalDefId, FxHashSet<ItemLocalId>>,
 
-    /// For each type and trait definition, maps type parameters
-    /// to the trait object lifetime defaults computed from them.
-    pub object_lifetime_defaults:
-        FxHashMap<LocalDefId, FxHashMap<ItemLocalId, Vec<ObjectLifetimeDefault>>>,
+    pub late_bound_vars: FxHashMap<LocalDefId, FxHashMap<ItemLocalId, Vec<ty::BoundVariableKind>>>,
 }
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 89ca8eed39a..fc9a2970e00 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -29,12 +29,6 @@ pub enum StabilityLevel {
     Stable,
 }
 
-impl StabilityLevel {
-    pub fn from_attr_level(level: &attr::StabilityLevel) -> Self {
-        if level.is_stable() { Stable } else { Unstable }
-    }
-}
-
 /// An entry in the `depr_map`.
 #[derive(Clone, HashStable, Debug)]
 pub struct DeprecationEntry {
diff --git a/compiler/rustc_middle/src/mir/abstract_const.rs b/compiler/rustc_middle/src/mir/abstract_const.rs
index b85f1e6e5de..776a777b1bd 100644
--- a/compiler/rustc_middle/src/mir/abstract_const.rs
+++ b/compiler/rustc_middle/src/mir/abstract_const.rs
@@ -18,3 +18,20 @@ pub enum Node<'tcx> {
     UnaryOp(mir::UnOp, NodeId),
     FunctionCall(NodeId, &'tcx [NodeId]),
 }
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
+pub enum NotConstEvaluatable {
+    Error(rustc_errors::ErrorReported),
+    MentionsInfer,
+    MentionsParam,
+}
+
+impl From<rustc_errors::ErrorReported> for NotConstEvaluatable {
+    fn from(e: rustc_errors::ErrorReported) -> NotConstEvaluatable {
+        NotConstEvaluatable::Error(e)
+    }
+}
+
+TrivialTypeFoldableAndLiftImpls! {
+    NotConstEvaluatable,
+}
diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs
index eae02a8cbfc..ddb1a84fe7b 100644
--- a/compiler/rustc_middle/src/mir/coverage.rs
+++ b/compiler/rustc_middle/src/mir/coverage.rs
@@ -41,8 +41,16 @@ rustc_index::newtype_index! {
 }
 
 impl CounterValueReference {
-    // Counters start at 1 to reserve 0 for ExpressionOperandId::ZERO.
+    /// Counters start at 1 to reserve 0 for ExpressionOperandId::ZERO.
     pub const START: Self = Self::from_u32(1);
+
+    /// Returns explicitly-requested zero-based version of the counter id, used
+    /// during codegen. LLVM expects zero-based indexes.
+    pub fn zero_based_index(&self) -> u32 {
+        let one_based_index = self.as_u32();
+        debug_assert!(one_based_index > 0);
+        one_based_index - 1
+    }
 }
 
 rustc_index::newtype_index! {
@@ -117,17 +125,9 @@ impl CoverageKind {
         }
     }
 
-    pub fn is_counter(&self) -> bool {
-        matches!(self, Self::Counter { .. })
-    }
-
     pub fn is_expression(&self) -> bool {
         matches!(self, Self::Expression { .. })
     }
-
-    pub fn is_unreachable(&self) -> bool {
-        *self == Self::Unreachable
-    }
 }
 
 impl Debug for CoverageKind {
@@ -183,3 +183,13 @@ pub enum Op {
     Subtract,
     Add,
 }
+
+impl Op {
+    pub fn is_add(&self) -> bool {
+        matches!(self, Self::Add)
+    }
+
+    pub fn is_subtract(&self) -> bool {
+        matches!(self, Self::Subtract)
+    }
+}
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index 48595f2badc..766d6a06f7e 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -339,7 +339,7 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra<Tag>> Allocation<Tag, Extra> {
         for dest in bytes {
             *dest = src.next().expect("iterator was shorter than it said it would be");
         }
-        src.next().expect_none("iterator was longer than it said it would be");
+        assert_matches!(src.next(), None, "iterator was longer than it said it would be");
         Ok(())
     }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index 1589ab28e40..cc0df127434 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -40,13 +40,13 @@ pub fn struct_error<'tcx>(tcx: TyCtxtAt<'tcx>, msg: &str) -> DiagnosticBuilder<'
     struct_span_err!(tcx.sess, tcx.span, E0080, "{}", msg)
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(InterpErrorInfo<'_>, 8);
 
 /// Packages the kind of error we got from the const code interpreter
 /// up with a Rust-level backtrace of where the error occurred.
-/// Thsese should always be constructed by calling `.into()` on
-/// a `InterpError`. In `librustc_mir::interpret`, we have `throw_err_*`
+/// These should always be constructed by calling `.into()` on
+/// a `InterpError`. In `rustc_mir::interpret`, we have `throw_err_*`
 /// macros for this.
 #[derive(Debug)]
 pub struct InterpErrorInfo<'tcx>(Box<InterpErrorInfoInner<'tcx>>);
@@ -444,7 +444,7 @@ impl dyn MachineStopType {
     }
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(InterpError<'_>, 72);
 
 pub enum InterpError<'tcx> {
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index 3e7b93b32a6..fa7c0670e8c 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -1,7 +1,7 @@
 use super::{ErrorHandled, EvalToConstValueResult, GlobalId};
 
 use crate::mir;
-use crate::ty::subst::{InternalSubsts, SubstsRef};
+use crate::ty::subst::InternalSubsts;
 use crate::ty::{self, TyCtxt};
 use rustc_hir::def_id::DefId;
 use rustc_span::Span;
@@ -35,14 +35,12 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn const_eval_resolve(
         self,
         param_env: ty::ParamEnv<'tcx>,
-        def: ty::WithOptConstParam<DefId>,
-        substs: SubstsRef<'tcx>,
-        promoted: Option<mir::Promoted>,
+        ct: ty::Unevaluated<'tcx>,
         span: Option<Span>,
     ) -> EvalToConstValueResult<'tcx> {
-        match ty::Instance::resolve_opt_const_arg(self, param_env, def, substs) {
+        match ty::Instance::resolve_opt_const_arg(self, param_env, ct.def, ct.substs) {
             Ok(Some(instance)) => {
-                let cid = GlobalId { instance, promoted };
+                let cid = GlobalId { instance, promoted: ct.promoted };
                 self.const_eval_global_id(param_env, cid, span)
             }
             Ok(None) => Err(ErrorHandled::TooGeneric),
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index 4bb39fe4a52..888777a9418 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -1,4 +1,4 @@
-use std::convert::TryFrom;
+use std::convert::{TryFrom, TryInto};
 use std::fmt;
 
 use rustc_apfloat::{
@@ -8,12 +8,12 @@ use rustc_apfloat::{
 use rustc_macros::HashStable;
 use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout};
 
-use crate::ty::{ParamEnv, ScalarInt, Ty, TyCtxt};
+use crate::ty::{Lift, ParamEnv, ScalarInt, Ty, TyCtxt};
 
 use super::{AllocId, Allocation, InterpResult, Pointer, PointerArithmetic};
 
 /// Represents the result of const evaluation via the `eval_to_allocation` query.
-#[derive(Clone, HashStable, TyEncodable, TyDecodable, Debug)]
+#[derive(Copy, Clone, HashStable, TyEncodable, TyDecodable, Debug, Hash, Eq, PartialEq)]
 pub struct ConstAlloc<'tcx> {
     // the value lives here, at offset 0, and that allocation definitely is a `AllocKind::Memory`
     // (so you can use `AllocMap::unwrap_memory`).
@@ -44,9 +44,30 @@ pub enum ConstValue<'tcx> {
     },
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(ConstValue<'_>, 32);
 
+impl From<Scalar> for ConstValue<'tcx> {
+    fn from(s: Scalar) -> Self {
+        Self::Scalar(s)
+    }
+}
+
+impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> {
+    type Lifted = ConstValue<'tcx>;
+    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ConstValue<'tcx>> {
+        Some(match self {
+            ConstValue::Scalar(s) => ConstValue::Scalar(s),
+            ConstValue::Slice { data, start, end } => {
+                ConstValue::Slice { data: tcx.lift(data)?, start, end }
+            }
+            ConstValue::ByRef { alloc, offset } => {
+                ConstValue::ByRef { alloc: tcx.lift(alloc)?, offset }
+            }
+        })
+    }
+}
+
 impl<'tcx> ConstValue<'tcx> {
     #[inline]
     pub fn try_to_scalar(&self) -> Option<Scalar> {
@@ -56,20 +77,20 @@ impl<'tcx> ConstValue<'tcx> {
         }
     }
 
+    pub fn try_to_scalar_int(&self) -> Option<ScalarInt> {
+        Some(self.try_to_scalar()?.assert_int())
+    }
+
     pub fn try_to_bits(&self, size: Size) -> Option<u128> {
-        self.try_to_scalar()?.to_bits(size).ok()
+        self.try_to_scalar_int()?.to_bits(size).ok()
     }
 
     pub fn try_to_bool(&self) -> Option<bool> {
-        match self.try_to_bits(Size::from_bytes(1))? {
-            0 => Some(false),
-            1 => Some(true),
-            _ => None,
-        }
+        self.try_to_scalar_int()?.try_into().ok()
     }
 
     pub fn try_to_machine_usize(&self, tcx: TyCtxt<'tcx>) -> Option<u64> {
-        Some(self.try_to_bits(tcx.data_layout.pointer_size)? as u64)
+        self.try_to_scalar_int()?.try_to_machine_usize(tcx).ok()
     }
 
     pub fn try_to_bits_for_ty(
@@ -111,7 +132,7 @@ pub enum Scalar<Tag = ()> {
     Ptr(Pointer<Tag>),
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(Scalar, 24);
 
 // We want the `Debug` output to be readable as it is used by `derive(Debug)` for
@@ -287,16 +308,6 @@ impl<'tcx, Tag> Scalar<Tag> {
     }
 
     #[inline]
-    pub fn from_i8(i: i8) -> Self {
-        Self::from_int(i, Size::from_bits(8))
-    }
-
-    #[inline]
-    pub fn from_i16(i: i16) -> Self {
-        Self::from_int(i, Size::from_bits(16))
-    }
-
-    #[inline]
     pub fn from_i32(i: i32) -> Self {
         Self::from_int(i, Size::from_bits(32))
     }
@@ -503,13 +514,20 @@ impl<Tag> From<Pointer<Tag>> for Scalar<Tag> {
     }
 }
 
+impl<Tag> From<ScalarInt> for Scalar<Tag> {
+    #[inline(always)]
+    fn from(ptr: ScalarInt) -> Self {
+        Scalar::Int(ptr)
+    }
+}
+
 #[derive(Clone, Copy, Eq, PartialEq, TyEncodable, TyDecodable, HashStable, Hash)]
 pub enum ScalarMaybeUninit<Tag = ()> {
     Scalar(Scalar<Tag>),
     Uninit,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(ScalarMaybeUninit, 24);
 
 impl<Tag> From<Scalar<Tag>> for ScalarMaybeUninit<Tag> {
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 4d8615a215f..99886821140 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -11,12 +11,12 @@ use crate::ty::fold::{TypeFoldable, TypeFolder, TypeVisitor};
 use crate::ty::print::{FmtPrinter, Printer};
 use crate::ty::subst::{Subst, SubstsRef};
 use crate::ty::{self, List, Ty, TyCtxt};
-use crate::ty::{AdtDef, InstanceDef, Region, UserTypeAnnotationIndex};
+use crate::ty::{AdtDef, InstanceDef, Region, ScalarInt, UserTypeAnnotationIndex};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::def_id::{DefId, CRATE_DEF_INDEX};
 use rustc_hir::{self, GeneratorKind};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{Size, VariantIdx};
 
 use polonius_engine::Atom;
 pub use rustc_ast::Mutability;
@@ -30,6 +30,7 @@ use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use std::borrow::Cow;
+use std::convert::TryInto;
 use std::fmt::{self, Debug, Display, Formatter, Write};
 use std::ops::{ControlFlow, Index, IndexMut};
 use std::slice;
@@ -378,24 +379,6 @@ impl<'tcx> Body<'tcx> {
         }
     }
 
-    /// Returns an iterator over all temporaries.
-    #[inline]
-    pub fn temps_iter<'a>(&'a self) -> impl Iterator<Item = Local> + 'a {
-        (self.arg_count + 1..self.local_decls.len()).filter_map(move |index| {
-            let local = Local::new(index);
-            if self.local_decls[local].is_user_variable() { None } else { Some(local) }
-        })
-    }
-
-    /// Returns an iterator over all user-declared locals.
-    #[inline]
-    pub fn vars_iter<'a>(&'a self) -> impl Iterator<Item = Local> + 'a {
-        (self.arg_count + 1..self.local_decls.len()).filter_map(move |index| {
-            let local = Local::new(index);
-            self.local_decls[local].is_user_variable().then_some(local)
-        })
-    }
-
     /// Returns an iterator over all user-declared mutable locals.
     #[inline]
     pub fn mut_vars_iter<'a>(&'a self) -> impl Iterator<Item = Local> + 'a {
@@ -951,7 +934,7 @@ pub struct LocalDecl<'tcx> {
 }
 
 // `LocalDecl` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(LocalDecl<'_>, 56);
 
 /// Extra information about a some locals that's used for diagnostics and for
@@ -1230,7 +1213,7 @@ pub enum InlineAsmOperand<'tcx> {
         out_place: Option<Place<'tcx>>,
     },
     Const {
-        value: Operand<'tcx>,
+        value: Box<Constant<'tcx>>,
     },
     SymFn {
         value: Box<Constant<'tcx>>,
@@ -1468,7 +1451,7 @@ pub struct Statement<'tcx> {
 }
 
 // `Statement` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(Statement<'_>, 32);
 
 impl Statement<'_> {
@@ -1499,7 +1482,7 @@ pub enum StatementKind<'tcx> {
     ///
     /// Note that this also is emitted for regular `let` bindings to ensure that locals that are
     /// never accessed still get some sanity checks for, e.g., `let x: ! = ..;`
-    FakeRead(FakeReadCause, Box<Place<'tcx>>),
+    FakeRead(Box<(FakeReadCause, Place<'tcx>)>),
 
     /// Write the discriminant for a variant to the enum Place.
     SetDiscriminant { place: Box<Place<'tcx>>, variant_index: VariantIdx },
@@ -1536,11 +1519,17 @@ pub enum StatementKind<'tcx> {
     AscribeUserType(Box<(Place<'tcx>, UserTypeProjection)>, ty::Variance),
 
     /// Marks the start of a "coverage region", injected with '-Zinstrument-coverage'. A
-    /// `CoverageInfo` statement carries metadata about the coverage region, used to inject a coverage
-    /// map into the binary. The `Counter` kind also generates executable code, to increment a
-    /// counter varible at runtime, each time the code region is executed.
+    /// `Coverage` statement carries metadata about the coverage region, used to inject a coverage
+    /// map into the binary. If `Coverage::kind` is a `Counter`, the statement also generates
+    /// executable code, to increment a counter variable at runtime, each time the code region is
+    /// executed.
     Coverage(Box<Coverage>),
 
+    /// Denotes a call to the intrinsic function copy_overlapping, where `src_dst` denotes the
+    /// memory being read from and written to(one field to save memory), and size
+    /// indicates how many bytes are being copied over.
+    CopyNonOverlapping(Box<CopyNonOverlapping<'tcx>>),
+
     /// No-op. Useful for deleting instructions without affecting statement indices.
     Nop,
 }
@@ -1586,7 +1575,12 @@ pub enum FakeReadCause {
 
     /// `let x: !; match x {}` doesn't generate any read of x so we need to
     /// generate a read of x to check that it is initialized and safe.
-    ForMatchedPlace,
+    ///
+    /// If a closure pattern matches a Place starting with an Upvar, then we introduce a
+    /// FakeRead for that Place outside the closure, in such a case this option would be
+    /// Some(closure_def_id).
+    /// Otherwise, the value of the optional DefId will be None.
+    ForMatchedPlace(Option<DefId>),
 
     /// A fake read of the RefWithinGuard version of a bind-by-value variable
     /// in a match guard to ensure that it's value hasn't change by the time
@@ -1605,7 +1599,12 @@ pub enum FakeReadCause {
     /// but in some cases it can affect the borrow checker, as in #53695.
     /// Therefore, we insert a "fake read" here to ensure that we get
     /// appropriate errors.
-    ForLet,
+    ///
+    /// If a closure pattern matches a Place starting with an Upvar, then we introduce a
+    /// FakeRead for that Place outside the closure, in such a case this option would be
+    /// Some(closure_def_id).
+    /// Otherwise, the value of the optional DefId will be None.
+    ForLet(Option<DefId>),
 
     /// If we have an index expression like
     ///
@@ -1629,7 +1628,9 @@ impl Debug for Statement<'_> {
         use self::StatementKind::*;
         match self.kind {
             Assign(box (ref place, ref rv)) => write!(fmt, "{:?} = {:?}", place, rv),
-            FakeRead(ref cause, ref place) => write!(fmt, "FakeRead({:?}, {:?})", cause, place),
+            FakeRead(box (ref cause, ref place)) => {
+                write!(fmt, "FakeRead({:?}, {:?})", cause, place)
+            }
             Retag(ref kind, ref place) => write!(
                 fmt,
                 "Retag({}{:?})",
@@ -1659,6 +1660,13 @@ impl Debug for Statement<'_> {
                     write!(fmt, "Coverage::{:?}", coverage.kind)
                 }
             }
+            CopyNonOverlapping(box crate::mir::CopyNonOverlapping {
+                ref src,
+                ref dst,
+                ref count,
+            }) => {
+                write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count)
+            }
             Nop => write!(fmt, "nop"),
         }
     }
@@ -1670,6 +1678,14 @@ pub struct Coverage {
     pub code_region: Option<CodeRegion>,
 }
 
+#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable)]
+pub struct CopyNonOverlapping<'tcx> {
+    pub src: Operand<'tcx>,
+    pub dst: Operand<'tcx>,
+    /// Number of elements to copy from src to dest, not bytes.
+    pub count: Operand<'tcx>,
+}
+
 ///////////////////////////////////////////////////////////////////////////
 // Places
 
@@ -1683,6 +1699,9 @@ pub struct Place<'tcx> {
     pub projection: &'tcx List<PlaceElem<'tcx>>,
 }
 
+#[cfg(target_arch = "x86_64")]
+static_assert_size!(Place<'_>, 16);
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[derive(TyEncodable, TyDecodable, HashStable)]
 pub enum ProjectionElem<V, T> {
@@ -1752,7 +1771,7 @@ impl<V, T> ProjectionElem<V, T> {
 pub type PlaceElem<'tcx> = ProjectionElem<Local, Ty<'tcx>>;
 
 // At least on 64 bit systems, `PlaceElem` should not be larger than two pointers.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(PlaceElem<'_>, 24);
 
 /// Alias for projections as they appear in `UserTypeProjection`, where we
@@ -1981,6 +2000,9 @@ pub enum Operand<'tcx> {
     Constant(Box<Constant<'tcx>>),
 }
 
+#[cfg(target_arch = "x86_64")]
+static_assert_size!(Operand<'_>, 24);
+
 impl<'tcx> Debug for Operand<'tcx> {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         use self::Operand::*;
@@ -2006,7 +2028,7 @@ impl<'tcx> Operand<'tcx> {
         Operand::Constant(box Constant {
             span,
             user_ty: None,
-            literal: ty::Const::zero_sized(tcx, ty),
+            literal: ConstantKind::Ty(ty::Const::zero_sized(tcx, ty)),
         })
     }
 
@@ -2037,7 +2059,7 @@ impl<'tcx> Operand<'tcx> {
         Operand::Constant(box Constant {
             span,
             user_ty: None,
-            literal: ty::Const::from_scalar(tcx, val, ty),
+            literal: ConstantKind::Val(val.into(), ty),
         })
     }
 
@@ -2096,8 +2118,8 @@ pub enum Rvalue<'tcx> {
 
     Cast(CastKind, Operand<'tcx>, Ty<'tcx>),
 
-    BinaryOp(BinOp, Operand<'tcx>, Operand<'tcx>),
-    CheckedBinaryOp(BinOp, Operand<'tcx>, Operand<'tcx>),
+    BinaryOp(BinOp, Box<(Operand<'tcx>, Operand<'tcx>)>),
+    CheckedBinaryOp(BinOp, Box<(Operand<'tcx>, Operand<'tcx>)>),
 
     NullaryOp(NullOp, Ty<'tcx>),
     UnaryOp(UnOp, Operand<'tcx>),
@@ -2116,6 +2138,9 @@ pub enum Rvalue<'tcx> {
     Aggregate(Box<AggregateKind<'tcx>>, Vec<Operand<'tcx>>),
 }
 
+#[cfg(target_arch = "x86_64")]
+static_assert_size!(Rvalue<'_>, 40);
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
 pub enum CastKind {
     Misc,
@@ -2139,6 +2164,9 @@ pub enum AggregateKind<'tcx> {
     Generator(DefId, SubstsRef<'tcx>, hir::Movability),
 }
 
+#[cfg(target_arch = "x86_64")]
+static_assert_size!(AggregateKind<'_>, 48);
+
 #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
 pub enum BinOp {
     /// The `+` operator (addition)
@@ -2215,8 +2243,8 @@ impl<'tcx> Debug for Rvalue<'tcx> {
             Cast(ref kind, ref place, ref ty) => {
                 write!(fmt, "{:?} as {:?} ({:?})", place, ty, kind)
             }
-            BinaryOp(ref op, ref a, ref b) => write!(fmt, "{:?}({:?}, {:?})", op, a, b),
-            CheckedBinaryOp(ref op, ref a, ref b) => {
+            BinaryOp(ref op, box (ref a, ref b)) => write!(fmt, "{:?}({:?}, {:?})", op, a, b),
+            CheckedBinaryOp(ref op, box (ref a, ref b)) => {
                 write!(fmt, "Checked{:?}({:?}, {:?})", op, a, b)
             }
             UnaryOp(ref op, ref a) => write!(fmt, "{:?}({:?})", op, a),
@@ -2295,7 +2323,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                             CtorKind::Fn => fmt_tuple(fmt, &name),
                             CtorKind::Fictive => {
                                 let mut struct_fmt = fmt.debug_struct(&name);
-                                for (field, place) in variant_def.fields.iter().zip(places) {
+                                for (field, place) in iter::zip(&variant_def.fields, places) {
                                     struct_fmt.field(&field.ident.as_str(), place);
                                 }
                                 struct_fmt.finish()
@@ -2319,7 +2347,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                             let mut struct_fmt = fmt.debug_struct(&name);
 
                             if let Some(upvars) = tcx.upvars_mentioned(def_id) {
-                                for (&var_id, place) in upvars.keys().zip(places) {
+                                for (&var_id, place) in iter::zip(upvars.keys(), places) {
                                     let var_name = tcx.hir().name(var_id);
                                     struct_fmt.field(&var_name.as_str(), place);
                                 }
@@ -2338,7 +2366,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                             let mut struct_fmt = fmt.debug_struct(&name);
 
                             if let Some(upvars) = tcx.upvars_mentioned(def_id) {
-                                for (&var_id, place) in upvars.keys().zip(places) {
+                                for (&var_id, place) in iter::zip(upvars.keys(), places) {
                                     let var_name = tcx.hir().name(var_id);
                                     struct_fmt.field(&var_name.as_str(), place);
                                 }
@@ -2373,12 +2401,22 @@ pub struct Constant<'tcx> {
     /// Needed for NLL to impose user-given type constraints.
     pub user_ty: Option<UserTypeAnnotationIndex>,
 
-    pub literal: &'tcx ty::Const<'tcx>,
+    pub literal: ConstantKind<'tcx>,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, TyEncodable, TyDecodable, Hash, HashStable, Debug)]
+#[derive(Lift)]
+pub enum ConstantKind<'tcx> {
+    /// This constant came from the type system
+    Ty(&'tcx ty::Const<'tcx>),
+    /// This constant cannot go back into the type system, as it represents
+    /// something the type system cannot handle (e.g. pointers).
+    Val(interpret::ConstValue<'tcx>, Ty<'tcx>),
 }
 
 impl Constant<'tcx> {
     pub fn check_static_ptr(&self, tcx: TyCtxt<'_>) -> Option<DefId> {
-        match self.literal.val.try_to_scalar() {
+        match self.literal.const_for_ty()?.val.try_to_scalar() {
             Some(Scalar::Ptr(ptr)) => match tcx.global_alloc(ptr.alloc_id) {
                 GlobalAlloc::Static(def_id) => {
                     assert!(!tcx.is_thread_local_static(def_id));
@@ -2389,6 +2427,94 @@ impl Constant<'tcx> {
             _ => None,
         }
     }
+    pub fn ty(&self) -> Ty<'tcx> {
+        self.literal.ty()
+    }
+}
+
+impl From<&'tcx ty::Const<'tcx>> for ConstantKind<'tcx> {
+    fn from(ct: &'tcx ty::Const<'tcx>) -> Self {
+        Self::Ty(ct)
+    }
+}
+
+impl ConstantKind<'tcx> {
+    /// Returns `None` if the constant is not trivially safe for use in the type system.
+    pub fn const_for_ty(&self) -> Option<&'tcx ty::Const<'tcx>> {
+        match self {
+            ConstantKind::Ty(c) => Some(c),
+            ConstantKind::Val(..) => None,
+        }
+    }
+
+    pub fn ty(&self) -> Ty<'tcx> {
+        match self {
+            ConstantKind::Ty(c) => c.ty,
+            ConstantKind::Val(_, ty) => ty,
+        }
+    }
+
+    #[inline]
+    pub fn try_to_value(self) -> Option<interpret::ConstValue<'tcx>> {
+        match self {
+            ConstantKind::Ty(c) => c.val.try_to_value(),
+            ConstantKind::Val(val, _) => Some(val),
+        }
+    }
+
+    #[inline]
+    pub fn try_to_scalar(self) -> Option<Scalar> {
+        self.try_to_value()?.try_to_scalar()
+    }
+
+    #[inline]
+    pub fn try_to_scalar_int(self) -> Option<ScalarInt> {
+        Some(self.try_to_value()?.try_to_scalar()?.assert_int())
+    }
+
+    #[inline]
+    pub fn try_to_bits(self, size: Size) -> Option<u128> {
+        self.try_to_scalar_int()?.to_bits(size).ok()
+    }
+
+    #[inline]
+    pub fn try_to_bool(self) -> Option<bool> {
+        self.try_to_scalar_int()?.try_into().ok()
+    }
+
+    #[inline]
+    pub fn try_eval_bits(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+        ty: Ty<'tcx>,
+    ) -> Option<u128> {
+        match self {
+            Self::Ty(ct) => ct.try_eval_bits(tcx, param_env, ty),
+            Self::Val(val, t) => {
+                assert_eq!(*t, ty);
+                let size =
+                    tcx.layout_of(param_env.with_reveal_all_normalized(tcx).and(ty)).ok()?.size;
+                val.try_to_bits(size)
+            }
+        }
+    }
+
+    #[inline]
+    pub fn try_eval_bool(&self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Option<bool> {
+        match self {
+            Self::Ty(ct) => ct.try_eval_bool(tcx, param_env),
+            Self::Val(val, _) => val.try_to_bool(),
+        }
+    }
+
+    #[inline]
+    pub fn try_eval_usize(&self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Option<u64> {
+        match self {
+            Self::Ty(ct) => ct.try_eval_usize(tcx, param_env),
+            Self::Val(val, _) => val.try_to_machine_usize(tcx),
+        }
+    }
 }
 
 /// A collection of projections into user types.
@@ -2574,11 +2700,20 @@ impl<'tcx> Debug for Constant<'tcx> {
 
 impl<'tcx> Display for Constant<'tcx> {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
-        match self.literal.ty.kind() {
+        match self.ty().kind() {
             ty::FnDef(..) => {}
             _ => write!(fmt, "const ")?,
         }
-        pretty_print_const(self.literal, fmt, true)
+        Display::fmt(&self.literal, fmt)
+    }
+}
+
+impl<'tcx> Display for ConstantKind<'tcx> {
+    fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
+        match *self {
+            ConstantKind::Ty(c) => pretty_print_const(c, fmt, true),
+            ConstantKind::Val(val, ty) => pretty_print_const_value(val, ty, fmt, true),
+        }
     }
 }
 
@@ -2597,6 +2732,23 @@ fn pretty_print_const(
     })
 }
 
+fn pretty_print_const_value(
+    val: interpret::ConstValue<'tcx>,
+    ty: Ty<'tcx>,
+    fmt: &mut Formatter<'_>,
+    print_types: bool,
+) -> fmt::Result {
+    use crate::ty::print::PrettyPrinter;
+    ty::tls::with(|tcx| {
+        let val = tcx.lift(val).unwrap();
+        let ty = tcx.lift(ty).unwrap();
+        let mut cx = FmtPrinter::new(tcx, fmt, Namespace::ValueNS);
+        cx.print_alloc_ids = true;
+        cx.pretty_print_const_value(val, ty, print_types)?;
+        Ok(())
+    })
+}
+
 impl<'tcx> graph::DirectedGraph for Body<'tcx> {
     type Node = BasicBlock;
 }
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index 6c2468b9ffe..92a1094bbcd 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -181,6 +181,11 @@ impl<'tcx> MonoItem<'tcx> {
         }
         .map(|hir_id| tcx.hir().span(hir_id))
     }
+
+    // Only used by rustc_codegen_cranelift
+    pub fn codegen_dep_node(&self, tcx: TyCtxt<'tcx>) -> DepNode {
+        crate::dep_graph::make_compile_mono_item(tcx, self)
+    }
 }
 
 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for MonoItem<'tcx> {
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index c293fbe4ef8..ad3baccf154 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -23,17 +23,9 @@ pub enum UnsafetyViolationKind {
     General,
     /// Permitted both in `const fn`s and regular `fn`s.
     GeneralAndConstFn,
-    /// Borrow of packed field.
-    /// Has to be handled as a lint for backwards compatibility.
-    BorrowPacked,
     /// Unsafe operation in an `unsafe fn` but outside an `unsafe` block.
     /// Has to be handled as a lint for backwards compatibility.
-    /// Should stay gated under `#![feature(unsafe_block_in_unsafe_fn)]`.
     UnsafeFn,
-    /// Borrow of packed field in an `unsafe fn` but outside an `unsafe` block.
-    /// Has to be handled as a lint for backwards compatibility.
-    /// Should stay gated under `#![feature(unsafe_block_in_unsafe_fn)]`.
-    UnsafeFnBorrowPacked,
 }
 
 #[derive(Copy, Clone, PartialEq, TyEncodable, TyDecodable, HashStable, Debug)]
@@ -439,18 +431,6 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline]
-    pub fn optimized_mir_or_const_arg_mir(
-        self,
-        def: ty::WithOptConstParam<DefId>,
-    ) -> &'tcx Body<'tcx> {
-        if let Some((did, param_did)) = def.as_const_arg() {
-            self.mir_for_ctfe_of_const_arg((did, param_did))
-        } else {
-            self.optimized_mir(def.did)
-        }
-    }
-
-    #[inline]
     pub fn mir_for_ctfe_opt_const_arg(self, def: ty::WithOptConstParam<DefId>) -> &'tcx Body<'tcx> {
         if let Some((did, param_did)) = def.as_const_arg() {
             self.mir_for_ctfe_of_const_arg((did, param_did))
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 92f46ef6a63..6e819145976 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -17,7 +17,7 @@ pub struct PlaceTy<'tcx> {
 }
 
 // At least on 64 bit systems, `PlaceTy` should not be larger than two or three pointers.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(PlaceTy<'_>, 16);
 
 impl<'tcx> PlaceTy<'tcx> {
@@ -182,12 +182,12 @@ impl<'tcx> Rvalue<'tcx> {
             }
             Rvalue::Len(..) => tcx.types.usize,
             Rvalue::Cast(.., ty) => ty,
-            Rvalue::BinaryOp(op, ref lhs, ref rhs) => {
+            Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => {
                 let lhs_ty = lhs.ty(local_decls, tcx);
                 let rhs_ty = rhs.ty(local_decls, tcx);
                 op.ty(tcx, lhs_ty, rhs_ty)
             }
-            Rvalue::CheckedBinaryOp(op, ref lhs, ref rhs) => {
+            Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => {
                 let lhs_ty = lhs.ty(local_decls, tcx);
                 let rhs_ty = rhs.ty(local_decls, tcx);
                 let ty = op.ty(tcx, lhs_ty, rhs_ty);
@@ -227,7 +227,7 @@ impl<'tcx> Operand<'tcx> {
     {
         match self {
             &Operand::Copy(ref l) | &Operand::Move(ref l) => l.ty(local_decls, tcx).ty,
-            &Operand::Constant(ref c) => c.literal.ty,
+            &Operand::Constant(ref c) => c.literal.ty(),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 887dbefa9f2..c8db4aeb449 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -67,7 +67,7 @@ impl SwitchTargets {
     ///
     /// Note that this may yield 0 elements. Only the `otherwise` branch is mandatory.
     pub fn iter(&self) -> SwitchTargetsIter<'_> {
-        SwitchTargetsIter { inner: self.values.iter().zip(self.targets.iter()) }
+        SwitchTargetsIter { inner: iter::zip(&self.values, &self.targets) }
     }
 
     /// Returns a slice with all possible jump targets (including the fallback target).
diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs
index 36e277d1a88..725448584dd 100644
--- a/compiler/rustc_middle/src/mir/traversal.rs
+++ b/compiler/rustc_middle/src/mir/traversal.rs
@@ -264,10 +264,6 @@ impl<'a, 'tcx> ReversePostorder<'a, 'tcx> {
 
         ReversePostorder { body, blocks, idx: len }
     }
-
-    pub fn reset(&mut self) {
-        self.idx = self.blocks.len();
-    }
 }
 
 pub fn reverse_postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> ReversePostorder<'a, 'tcx> {
diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs
index da8e189ba9d..f3124e5bf42 100644
--- a/compiler/rustc_middle/src/mir/type_foldable.rs
+++ b/compiler/rustc_middle/src/mir/type_foldable.rs
@@ -181,9 +181,11 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
             AddressOf(mutability, place) => AddressOf(mutability, place.fold_with(folder)),
             Len(place) => Len(place.fold_with(folder)),
             Cast(kind, op, ty) => Cast(kind, op.fold_with(folder), ty.fold_with(folder)),
-            BinaryOp(op, rhs, lhs) => BinaryOp(op, rhs.fold_with(folder), lhs.fold_with(folder)),
-            CheckedBinaryOp(op, rhs, lhs) => {
-                CheckedBinaryOp(op, rhs.fold_with(folder), lhs.fold_with(folder))
+            BinaryOp(op, box (rhs, lhs)) => {
+                BinaryOp(op, box (rhs.fold_with(folder), lhs.fold_with(folder)))
+            }
+            CheckedBinaryOp(op, box (rhs, lhs)) => {
+                CheckedBinaryOp(op, box (rhs.fold_with(folder), lhs.fold_with(folder)))
             }
             UnaryOp(op, val) => UnaryOp(op, val.fold_with(folder)),
             Discriminant(place) => Discriminant(place.fold_with(folder)),
@@ -227,7 +229,7 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
                 op.visit_with(visitor)?;
                 ty.visit_with(visitor)
             }
-            BinaryOp(_, ref rhs, ref lhs) | CheckedBinaryOp(_, ref rhs, ref lhs) => {
+            BinaryOp(_, box (ref rhs, ref lhs)) | CheckedBinaryOp(_, box (ref rhs, ref lhs)) => {
                 rhs.visit_with(visitor)?;
                 lhs.visit_with(visitor)
             }
@@ -340,6 +342,28 @@ impl<'tcx> TypeFoldable<'tcx> for Constant<'tcx> {
         }
     }
     fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.literal.visit_with(visitor)
+        self.literal.visit_with(visitor)?;
+        self.user_ty.visit_with(visitor)
+    }
+}
+
+impl<'tcx> TypeFoldable<'tcx> for ConstantKind<'tcx> {
+    #[inline(always)]
+    fn fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self {
+        folder.fold_mir_const(self)
+    }
+
+    fn super_fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self {
+        match self {
+            ConstantKind::Ty(c) => ConstantKind::Ty(c.fold_with(folder)),
+            ConstantKind::Val(v, t) => ConstantKind::Val(v, t.fold_with(folder)),
+        }
+    }
+
+    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        match *self {
+            ConstantKind::Ty(c) => c.visit_with(visitor),
+            ConstantKind::Val(_, t) => t.visit_with(visitor),
+        }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index 3e92844c4f9..5516a045c1d 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -380,7 +380,7 @@ macro_rules! make_mir_visitor {
                     ) => {
                         self.visit_assign(place, rvalue, location);
                     }
-                    StatementKind::FakeRead(_, place) => {
+                    StatementKind::FakeRead(box (_, place)) => {
                         self.visit_place(
                             place,
                             PlaceContext::NonMutatingUse(NonMutatingUseContext::Inspect),
@@ -436,6 +436,15 @@ macro_rules! make_mir_visitor {
                             location
                         )
                     }
+                    StatementKind::CopyNonOverlapping(box crate::mir::CopyNonOverlapping{
+                      ref $($mutability)? src,
+                      ref $($mutability)? dst,
+                      ref $($mutability)? count,
+                    }) => {
+                      self.visit_operand(src, location);
+                      self.visit_operand(dst, location);
+                      self.visit_operand(count, location)
+                    }
                     StatementKind::Nop => {}
                 }
             }
@@ -575,8 +584,7 @@ macro_rules! make_mir_visitor {
                     } => {
                         for op in operands {
                             match op {
-                                InlineAsmOperand::In { value, .. }
-                                | InlineAsmOperand::Const { value } => {
+                                InlineAsmOperand::In { value, .. } => {
                                     self.visit_operand(value, location);
                                 }
                                 InlineAsmOperand::Out { place, .. } => {
@@ -598,7 +606,8 @@ macro_rules! make_mir_visitor {
                                         );
                                     }
                                 }
-                                InlineAsmOperand::SymFn { value } => {
+                                InlineAsmOperand::Const { value }
+                                | InlineAsmOperand::SymFn { value } => {
                                     self.visit_constant(value, location);
                                 }
                                 InlineAsmOperand::SymStatic { def_id: _ } => {}
@@ -687,8 +696,8 @@ macro_rules! make_mir_visitor {
                         self.visit_ty(ty, TyContext::Location(location));
                     }
 
-                    Rvalue::BinaryOp(_bin_op, lhs, rhs)
-                    | Rvalue::CheckedBinaryOp(_bin_op, lhs, rhs) => {
+                    Rvalue::BinaryOp(_bin_op, box(lhs, rhs))
+                    | Rvalue::CheckedBinaryOp(_bin_op, box(lhs, rhs)) => {
                         self.visit_operand(lhs, location);
                         self.visit_operand(rhs, location);
                     }
@@ -862,7 +871,10 @@ macro_rules! make_mir_visitor {
 
                 self.visit_span(span);
                 drop(user_ty); // no visit method for this
-                self.visit_const(literal, location);
+                match literal {
+                    ConstantKind::Ty(ct) => self.visit_const(ct, location),
+                    ConstantKind::Val(_, t) => self.visit_ty(t, TyContext::Location(location)),
+                }
             }
 
             fn super_span(&mut self, _span: & $($mutability)? Span) {
@@ -1235,12 +1247,6 @@ impl PlaceContext {
         matches!(self, PlaceContext::MutatingUse(..))
     }
 
-    /// Returns `true` if this place context represents a use that does not change the value.
-    #[inline]
-    pub fn is_nonmutating_use(&self) -> bool {
-        matches!(self, PlaceContext::NonMutatingUse(..))
-    }
-
     /// Returns `true` if this place context represents a use.
     #[inline]
     pub fn is_use(&self) -> bool {
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index a8600af1de2..08fa12aa371 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -61,6 +61,15 @@ rustc_queries! {
         desc { |tcx| "HIR owner items in `{}`", tcx.def_path_str(key.to_def_id()) }
     }
 
+    /// Gives access to the HIR attributes inside the HIR owner `key`.
+    ///
+    /// This can be conveniently accessed by methods on `tcx.hir()`.
+    /// Avoid calling this query directly.
+    query hir_attrs(key: LocalDefId) -> rustc_middle::hir::AttributeMap<'tcx> {
+        eval_always
+        desc { |tcx| "HIR owner attributes in `{}`", tcx.def_path_str(key.to_def_id()) }
+    }
+
     /// Computes the `DefId` of the corresponding const parameter in case the `key` is a
     /// const argument and returns `None` otherwise.
     ///
@@ -84,6 +93,12 @@ rustc_queries! {
         desc { |tcx| "computing the optional const parameter of `{}`", tcx.def_path_str(key.to_def_id()) }
     }
 
+    /// Given the def_id of a const-generic parameter, computes the associated default const
+    /// parameter. e.g. `fn example<const N: usize=3>` called on `N` would return `3`.
+    query const_param_default(param: DefId) -> &'tcx ty::Const<'tcx> {
+        desc { |tcx| "compute const default for a given parameter `{}`", tcx.def_path_str(param)  }
+    }
+
     /// Records the type of every item.
     query type_of(key: DefId) -> Ty<'tcx> {
         desc { |tcx| "computing type of `{}`", tcx.def_path_str(key) }
@@ -319,7 +334,10 @@ rustc_queries! {
 
     /// Returns the name of the file that contains the function body, if instrumented for coverage.
     query covered_file_name(key: DefId) -> Option<Symbol> {
-        desc { |tcx| "retrieving the covered file name, if instrumented, for `{}`", tcx.def_path_str(key) }
+        desc {
+            |tcx| "retrieving the covered file name, if instrumented, for `{}`",
+            tcx.def_path_str(key)
+        }
         storage(ArenaCacheSelector<'tcx>)
         cache_on_disk_if { key.is_local() }
     }
@@ -327,7 +345,10 @@ rustc_queries! {
     /// Returns the `CodeRegions` for a function that has instrumented coverage, in case the
     /// function was optimized out before codegen, and before being added to the Coverage Map.
     query covered_code_regions(key: DefId) -> Vec<&'tcx mir::coverage::CodeRegion> {
-        desc { |tcx| "retrieving the covered `CodeRegion`s, if instrumented, for `{}`", tcx.def_path_str(key) }
+        desc {
+            |tcx| "retrieving the covered `CodeRegion`s, if instrumented, for `{}`",
+            tcx.def_path_str(key)
+        }
         storage(ArenaCacheSelector<'tcx>)
         cache_on_disk_if { key.is_local() }
     }
@@ -543,7 +564,7 @@ rustc_queries! {
     }
 
     /// Collects the associated items defined on a trait or impl.
-    query associated_items(key: DefId) -> ty::AssociatedItems<'tcx> {
+    query associated_items(key: DefId) -> ty::AssocItems<'tcx> {
         storage(ArenaCacheSelector<'tcx>)
         desc { |tcx| "collecting associated items of {}", tcx.def_path_str(key) }
     }
@@ -776,6 +797,14 @@ rustc_queries! {
         cache_on_disk_if { true }
     }
 
+    /// Convert an evaluated constant to a type level constant or
+    /// return `None` if that is not possible.
+    query const_to_valtree(
+        key: ty::ParamEnvAnd<'tcx, ConstAlloc<'tcx>>
+    ) -> Option<ty::ValTree<'tcx>> {
+        desc { "destructure constant" }
+    }
+
     /// Destructure a constant ADT or array into its variant index and its
     /// field values.
     query destructure_const(
@@ -976,6 +1005,10 @@ rustc_queries! {
     query is_freeze_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` is freeze", env.value }
     }
+    /// Query backing `TyS::is_unpin`.
+    query is_unpin_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
+        desc { "computing whether `{}` is `Unpin`", env.value }
+    }
     /// Query backing `TyS::needs_drop`.
     query needs_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` needs drop", env.value }
@@ -1022,6 +1055,8 @@ rustc_queries! {
         desc { "checking if the crate is_compiler_builtins" }
     }
     query has_global_allocator(_: CrateNum) -> bool {
+        // This query depends on untracked global state in CStore
+        eval_always
         fatal_cycle
         desc { "checking if the crate has_global_allocator" }
     }
@@ -1220,13 +1255,19 @@ rustc_queries! {
         desc { |tcx| "native_library_kind({})", tcx.def_path_str(def_id) }
     }
 
-    query link_args(_: CrateNum) -> Lrc<Vec<String>> {
-        eval_always
-        desc { "looking up link arguments for a crate" }
+    /// Does lifetime resolution, but does not descend into trait items. This
+    /// should only be used for resolving lifetimes of on trait definitions,
+    /// and is used to avoid cycles. Importantly, `resolve_lifetimes` still visits
+    /// the same lifetimes and is responsible for diagnostics.
+    /// See `rustc_resolve::late::lifetimes for details.
+    query resolve_lifetimes_trait_definition(_: LocalDefId) -> ResolveLifetimes {
+        storage(ArenaCacheSelector<'tcx>)
+        desc { "resolving lifetimes for a trait definition" }
     }
-
-    /// Lifetime resolution. See `middle::resolve_lifetimes`.
-    query resolve_lifetimes(_: CrateNum) -> ResolveLifetimes {
+    /// Does lifetime resolution on items. Importantly, we can't resolve
+    /// lifetimes directly on things like trait methods, because of trait params.
+    /// See `rustc_resolve::late::lifetimes for details.
+    query resolve_lifetimes(_: LocalDefId) -> ResolveLifetimes {
         storage(ArenaCacheSelector<'tcx>)
         desc { "resolving lifetimes" }
     }
@@ -1238,9 +1279,17 @@ rustc_queries! {
         Option<(LocalDefId, &'tcx FxHashSet<ItemLocalId>)> {
         desc { "testing if a region is late bound" }
     }
+    /// For a given item (like a struct), gets the default lifetimes to be used
+    /// for each parameter if a trait object were to be passed for that parameter.
+    /// For example, for `struct Foo<'a, T, U>`, this would be `['static, 'static]`.
+    /// For `struct Foo<'a, T: 'a, U>`, this would instead be `['a, 'static]`.
     query object_lifetime_defaults_map(_: LocalDefId)
-        -> Option<&'tcx FxHashMap<ItemLocalId, Vec<ObjectLifetimeDefault>>> {
-        desc { "looking up lifetime defaults for a region" }
+        -> Option<Vec<ObjectLifetimeDefault>> {
+        desc { "looking up lifetime defaults for a region on an item" }
+    }
+    query late_bound_vars_map(_: LocalDefId)
+        -> Option<&'tcx FxHashMap<ItemLocalId, Vec<ty::BoundVariableKind>>> {
+        desc { "looking up late bound vars" }
     }
 
     query visibility(def_id: DefId) -> ty::Visibility {
@@ -1269,6 +1318,8 @@ rustc_queries! {
         desc { |tcx| "collecting child items of `{}`", tcx.def_path_str(def_id) }
     }
     query extern_mod_stmt_cnum(def_id: LocalDefId) -> Option<CrateNum> {
+        // This depends on untracked global state (`tcx.extern_crate_map`)
+        eval_always
         desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id.to_def_id()) }
     }
 
@@ -1388,6 +1439,14 @@ rustc_queries! {
     query is_codegened_item(def_id: DefId) -> bool {
         desc { |tcx| "determining whether `{}` needs codegen", tcx.def_path_str(def_id) }
     }
+
+    /// All items participating in code generation together with items inlined into them.
+    query codegened_and_inlined_items(_: CrateNum)
+        -> &'tcx DefIdSet {
+        eval_always
+       desc { "codegened_and_inlined_items" }
+    }
+
     query codegen_unit(_: Symbol) -> &'tcx CodegenUnit<'tcx> {
         desc { "codegen_unit" }
     }
@@ -1424,6 +1483,13 @@ rustc_queries! {
         desc { "normalizing `{}`", goal.value }
     }
 
+    /// Do not call this query directly: invoke `normalize_erasing_regions` instead.
+    query normalize_mir_const_after_erasing_regions(
+        goal: ParamEnvAnd<'tcx, mir::ConstantKind<'tcx>>
+    ) -> mir::ConstantKind<'tcx> {
+        desc { "normalizing `{}`", goal.value }
+    }
+
     query implied_outlives_bounds(
         goal: CanonicalTyGoal<'tcx>
     ) -> Result<
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 9deeaf462d6..00dec3b355f 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -9,7 +9,7 @@ pub mod specialization_graph;
 mod structural_impls;
 
 use crate::infer::canonical::Canonical;
-use crate::mir::interpret::ErrorHandled;
+use crate::mir::abstract_const::NotConstEvaluatable;
 use crate::ty::subst::SubstsRef;
 use crate::ty::{self, AdtKind, Ty, TyCtxt};
 
@@ -323,6 +323,9 @@ pub enum ObligationCauseCode<'tcx> {
 
     /// #[feature(trivial_bounds)] is not enabled
     TrivialBound,
+
+    /// If `X` is the concrete type of an opaque type `impl Y`, then `X` must implement `Y`
+    OpaqueType,
 }
 
 impl ObligationCauseCode<'_> {
@@ -340,8 +343,8 @@ impl ObligationCauseCode<'_> {
 }
 
 // `ObligationCauseCode` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
-static_assert_size!(ObligationCauseCode<'_>, 32);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+static_assert_size!(ObligationCauseCode<'_>, 40);
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum StatementAsExpression {
@@ -398,7 +401,7 @@ pub enum SelectionError<'tcx> {
         ty::error::TypeError<'tcx>,
     ),
     TraitNotObjectSafe(DefId),
-    ConstEvalFailure(ErrorHandled),
+    NotConstEvaluatable(NotConstEvaluatable),
     Overflow,
 }
 
diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs
index f9cadb3bb2d..b0ab0c9ae52 100644
--- a/compiler/rustc_middle/src/traits/query.rs
+++ b/compiler/rustc_middle/src/traits/query.rs
@@ -3,7 +3,7 @@
 //! which makes a canonical query by replacing unbound inference
 //! variables and regions, so that results can be reused more broadly.
 //! The providers for the queries defined here can be found in
-//! `librustc_traits`.
+//! `rustc_traits`.
 
 use crate::ich::StableHashingContext;
 use crate::infer::canonical::{Canonical, QueryResponse};
@@ -44,24 +44,12 @@ pub mod type_op {
         pub b: Ty<'tcx>,
     }
 
-    impl<'tcx> Eq<'tcx> {
-        pub fn new(a: Ty<'tcx>, b: Ty<'tcx>) -> Self {
-            Self { a, b }
-        }
-    }
-
     #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, HashStable, TypeFoldable, Lift)]
     pub struct Subtype<'tcx> {
         pub sub: Ty<'tcx>,
         pub sup: Ty<'tcx>,
     }
 
-    impl<'tcx> Subtype<'tcx> {
-        pub fn new(sub: Ty<'tcx>, sup: Ty<'tcx>) -> Self {
-            Self { sub, sup }
-        }
-    }
-
     #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, HashStable, TypeFoldable, Lift)]
     pub struct ProvePredicate<'tcx> {
         pub predicate: Predicate<'tcx>,
diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs
index a5962e3b3ba..8e2c79701af 100644
--- a/compiler/rustc_middle/src/ty/_match.rs
+++ b/compiler/rustc_middle/src/ty/_match.rs
@@ -112,9 +112,9 @@ impl TypeRelation<'tcx> for Match<'tcx> {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs
new file mode 100644
index 00000000000..95159ea46ae
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/adt.rs
@@ -0,0 +1,482 @@
+use crate::ich::StableHashingContext;
+use crate::mir::interpret::ErrorHandled;
+use crate::ty;
+use crate::ty::util::{Discr, IntTypeExt};
+use rustc_data_structures::captures::Captures;
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_errors::ErrorReported;
+use rustc_hir::def::{DefKind, Res};
+use rustc_hir::def_id::DefId;
+use rustc_index::vec::{Idx, IndexVec};
+use rustc_serialize::{self, Encodable, Encoder};
+use rustc_session::DataTypeKind;
+use rustc_span::symbol::sym;
+use rustc_target::abi::VariantIdx;
+
+use std::cell::RefCell;
+use std::cmp::Ordering;
+use std::hash::{Hash, Hasher};
+use std::ops::Range;
+use std::{ptr, str};
+
+use super::{
+    Destructor, FieldDef, GenericPredicates, ReprOptions, Ty, TyCtxt, VariantDef, VariantDiscr,
+};
+
+#[derive(Clone, HashStable, Debug)]
+pub struct AdtSizedConstraint<'tcx>(pub &'tcx [Ty<'tcx>]);
+
+bitflags! {
+    #[derive(HashStable)]
+    pub struct AdtFlags: u32 {
+        const NO_ADT_FLAGS        = 0;
+        /// Indicates whether the ADT is an enum.
+        const IS_ENUM             = 1 << 0;
+        /// Indicates whether the ADT is a union.
+        const IS_UNION            = 1 << 1;
+        /// Indicates whether the ADT is a struct.
+        const IS_STRUCT           = 1 << 2;
+        /// Indicates whether the ADT is a struct and has a constructor.
+        const HAS_CTOR            = 1 << 3;
+        /// Indicates whether the type is `PhantomData`.
+        const IS_PHANTOM_DATA     = 1 << 4;
+        /// Indicates whether the type has a `#[fundamental]` attribute.
+        const IS_FUNDAMENTAL      = 1 << 5;
+        /// Indicates whether the type is `Box`.
+        const IS_BOX              = 1 << 6;
+        /// Indicates whether the type is `ManuallyDrop`.
+        const IS_MANUALLY_DROP    = 1 << 7;
+        /// Indicates whether the variant list of this ADT is `#[non_exhaustive]`.
+        /// (i.e., this flag is never set unless this ADT is an enum).
+        const IS_VARIANT_LIST_NON_EXHAUSTIVE = 1 << 8;
+    }
+}
+
+/// The definition of a user-defined type, e.g., a `struct`, `enum`, or `union`.
+///
+/// These are all interned (by `alloc_adt_def`) into the global arena.
+///
+/// The initialism *ADT* stands for an [*algebraic data type (ADT)*][adt].
+/// This is slightly wrong because `union`s are not ADTs.
+/// Moreover, Rust only allows recursive data types through indirection.
+///
+/// [adt]: https://en.wikipedia.org/wiki/Algebraic_data_type
+pub struct AdtDef {
+    /// The `DefId` of the struct, enum or union item.
+    pub did: DefId,
+    /// Variants of the ADT. If this is a struct or union, then there will be a single variant.
+    pub variants: IndexVec<VariantIdx, VariantDef>,
+    /// Flags of the ADT (e.g., is this a struct? is this non-exhaustive?).
+    flags: AdtFlags,
+    /// Repr options provided by the user.
+    pub repr: ReprOptions,
+}
+
+impl PartialOrd for AdtDef {
+    fn partial_cmp(&self, other: &AdtDef) -> Option<Ordering> {
+        Some(self.cmp(&other))
+    }
+}
+
+/// There should be only one AdtDef for each `did`, therefore
+/// it is fine to implement `Ord` only based on `did`.
+impl Ord for AdtDef {
+    fn cmp(&self, other: &AdtDef) -> Ordering {
+        self.did.cmp(&other.did)
+    }
+}
+
+impl PartialEq for AdtDef {
+    // `AdtDef`s are always interned, and this is part of `TyS` equality.
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        ptr::eq(self, other)
+    }
+}
+
+impl Eq for AdtDef {}
+
+impl Hash for AdtDef {
+    #[inline]
+    fn hash<H: Hasher>(&self, s: &mut H) {
+        (self as *const AdtDef).hash(s)
+    }
+}
+
+impl<S: Encoder> Encodable<S> for AdtDef {
+    fn encode(&self, s: &mut S) -> Result<(), S::Error> {
+        self.did.encode(s)
+    }
+}
+
+impl<'a> HashStable<StableHashingContext<'a>> for AdtDef {
+    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
+        thread_local! {
+            static CACHE: RefCell<FxHashMap<usize, Fingerprint>> = Default::default();
+        }
+
+        let hash: Fingerprint = CACHE.with(|cache| {
+            let addr = self as *const AdtDef as usize;
+            *cache.borrow_mut().entry(addr).or_insert_with(|| {
+                let ty::AdtDef { did, ref variants, ref flags, ref repr } = *self;
+
+                let mut hasher = StableHasher::new();
+                did.hash_stable(hcx, &mut hasher);
+                variants.hash_stable(hcx, &mut hasher);
+                flags.hash_stable(hcx, &mut hasher);
+                repr.hash_stable(hcx, &mut hasher);
+
+                hasher.finish()
+            })
+        });
+
+        hash.hash_stable(hcx, hasher);
+    }
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
+pub enum AdtKind {
+    Struct,
+    Union,
+    Enum,
+}
+
+impl Into<DataTypeKind> for AdtKind {
+    fn into(self) -> DataTypeKind {
+        match self {
+            AdtKind::Struct => DataTypeKind::Struct,
+            AdtKind::Union => DataTypeKind::Union,
+            AdtKind::Enum => DataTypeKind::Enum,
+        }
+    }
+}
+
+impl<'tcx> AdtDef {
+    /// Creates a new `AdtDef`.
+    pub(super) fn new(
+        tcx: TyCtxt<'_>,
+        did: DefId,
+        kind: AdtKind,
+        variants: IndexVec<VariantIdx, VariantDef>,
+        repr: ReprOptions,
+    ) -> Self {
+        debug!("AdtDef::new({:?}, {:?}, {:?}, {:?})", did, kind, variants, repr);
+        let mut flags = AdtFlags::NO_ADT_FLAGS;
+
+        if kind == AdtKind::Enum && tcx.has_attr(did, sym::non_exhaustive) {
+            debug!("found non-exhaustive variant list for {:?}", did);
+            flags = flags | AdtFlags::IS_VARIANT_LIST_NON_EXHAUSTIVE;
+        }
+
+        flags |= match kind {
+            AdtKind::Enum => AdtFlags::IS_ENUM,
+            AdtKind::Union => AdtFlags::IS_UNION,
+            AdtKind::Struct => AdtFlags::IS_STRUCT,
+        };
+
+        if kind == AdtKind::Struct && variants[VariantIdx::new(0)].ctor_def_id.is_some() {
+            flags |= AdtFlags::HAS_CTOR;
+        }
+
+        let attrs = tcx.get_attrs(did);
+        if tcx.sess.contains_name(&attrs, sym::fundamental) {
+            flags |= AdtFlags::IS_FUNDAMENTAL;
+        }
+        if Some(did) == tcx.lang_items().phantom_data() {
+            flags |= AdtFlags::IS_PHANTOM_DATA;
+        }
+        if Some(did) == tcx.lang_items().owned_box() {
+            flags |= AdtFlags::IS_BOX;
+        }
+        if Some(did) == tcx.lang_items().manually_drop() {
+            flags |= AdtFlags::IS_MANUALLY_DROP;
+        }
+
+        AdtDef { did, variants, flags, repr }
+    }
+
+    /// Returns `true` if this is a struct.
+    #[inline]
+    pub fn is_struct(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_STRUCT)
+    }
+
+    /// Returns `true` if this is a union.
+    #[inline]
+    pub fn is_union(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_UNION)
+    }
+
+    /// Returns `true` if this is a enum.
+    #[inline]
+    pub fn is_enum(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_ENUM)
+    }
+
+    /// Returns `true` if the variant list of this ADT is `#[non_exhaustive]`.
+    #[inline]
+    pub fn is_variant_list_non_exhaustive(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_VARIANT_LIST_NON_EXHAUSTIVE)
+    }
+
+    /// Returns the kind of the ADT.
+    #[inline]
+    pub fn adt_kind(&self) -> AdtKind {
+        if self.is_enum() {
+            AdtKind::Enum
+        } else if self.is_union() {
+            AdtKind::Union
+        } else {
+            AdtKind::Struct
+        }
+    }
+
+    /// Returns a description of this abstract data type.
+    pub fn descr(&self) -> &'static str {
+        match self.adt_kind() {
+            AdtKind::Struct => "struct",
+            AdtKind::Union => "union",
+            AdtKind::Enum => "enum",
+        }
+    }
+
+    /// Returns a description of a variant of this abstract data type.
+    #[inline]
+    pub fn variant_descr(&self) -> &'static str {
+        match self.adt_kind() {
+            AdtKind::Struct => "struct",
+            AdtKind::Union => "union",
+            AdtKind::Enum => "variant",
+        }
+    }
+
+    /// If this function returns `true`, it implies that `is_struct` must return `true`.
+    #[inline]
+    pub fn has_ctor(&self) -> bool {
+        self.flags.contains(AdtFlags::HAS_CTOR)
+    }
+
+    /// Returns `true` if this type is `#[fundamental]` for the purposes
+    /// of coherence checking.
+    #[inline]
+    pub fn is_fundamental(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_FUNDAMENTAL)
+    }
+
+    /// Returns `true` if this is `PhantomData<T>`.
+    #[inline]
+    pub fn is_phantom_data(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_PHANTOM_DATA)
+    }
+
+    /// Returns `true` if this is Box<T>.
+    #[inline]
+    pub fn is_box(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_BOX)
+    }
+
+    /// Returns `true` if this is `ManuallyDrop<T>`.
+    #[inline]
+    pub fn is_manually_drop(&self) -> bool {
+        self.flags.contains(AdtFlags::IS_MANUALLY_DROP)
+    }
+
+    /// Returns `true` if this type has a destructor.
+    pub fn has_dtor(&self, tcx: TyCtxt<'tcx>) -> bool {
+        self.destructor(tcx).is_some()
+    }
+
+    /// Asserts this is a struct or union and returns its unique variant.
+    pub fn non_enum_variant(&self) -> &VariantDef {
+        assert!(self.is_struct() || self.is_union());
+        &self.variants[VariantIdx::new(0)]
+    }
+
+    #[inline]
+    pub fn predicates(&self, tcx: TyCtxt<'tcx>) -> GenericPredicates<'tcx> {
+        tcx.predicates_of(self.did)
+    }
+
+    /// Returns an iterator over all fields contained
+    /// by this ADT.
+    #[inline]
+    pub fn all_fields(&self) -> impl Iterator<Item = &FieldDef> + Clone {
+        self.variants.iter().flat_map(|v| v.fields.iter())
+    }
+
+    /// Whether the ADT lacks fields. Note that this includes uninhabited enums,
+    /// e.g., `enum Void {}` is considered payload free as well.
+    pub fn is_payloadfree(&self) -> bool {
+        self.variants.iter().all(|v| v.fields.is_empty())
+    }
+
+    /// Return a `VariantDef` given a variant id.
+    pub fn variant_with_id(&self, vid: DefId) -> &VariantDef {
+        self.variants.iter().find(|v| v.def_id == vid).expect("variant_with_id: unknown variant")
+    }
+
+    /// Return a `VariantDef` given a constructor id.
+    pub fn variant_with_ctor_id(&self, cid: DefId) -> &VariantDef {
+        self.variants
+            .iter()
+            .find(|v| v.ctor_def_id == Some(cid))
+            .expect("variant_with_ctor_id: unknown variant")
+    }
+
+    /// Return the index of `VariantDef` given a variant id.
+    pub fn variant_index_with_id(&self, vid: DefId) -> VariantIdx {
+        self.variants
+            .iter_enumerated()
+            .find(|(_, v)| v.def_id == vid)
+            .expect("variant_index_with_id: unknown variant")
+            .0
+    }
+
+    /// Return the index of `VariantDef` given a constructor id.
+    pub fn variant_index_with_ctor_id(&self, cid: DefId) -> VariantIdx {
+        self.variants
+            .iter_enumerated()
+            .find(|(_, v)| v.ctor_def_id == Some(cid))
+            .expect("variant_index_with_ctor_id: unknown variant")
+            .0
+    }
+
+    pub fn variant_of_res(&self, res: Res) -> &VariantDef {
+        match res {
+            Res::Def(DefKind::Variant, vid) => self.variant_with_id(vid),
+            Res::Def(DefKind::Ctor(..), cid) => self.variant_with_ctor_id(cid),
+            Res::Def(DefKind::Struct, _)
+            | Res::Def(DefKind::Union, _)
+            | Res::Def(DefKind::TyAlias, _)
+            | Res::Def(DefKind::AssocTy, _)
+            | Res::SelfTy(..)
+            | Res::SelfCtor(..) => self.non_enum_variant(),
+            _ => bug!("unexpected res {:?} in variant_of_res", res),
+        }
+    }
+
+    #[inline]
+    pub fn eval_explicit_discr(&self, tcx: TyCtxt<'tcx>, expr_did: DefId) -> Option<Discr<'tcx>> {
+        assert!(self.is_enum());
+        let param_env = tcx.param_env(expr_did);
+        let repr_type = self.repr.discr_type();
+        match tcx.const_eval_poly(expr_did) {
+            Ok(val) => {
+                let ty = repr_type.to_ty(tcx);
+                if let Some(b) = val.try_to_bits_for_ty(tcx, param_env, ty) {
+                    trace!("discriminants: {} ({:?})", b, repr_type);
+                    Some(Discr { val: b, ty })
+                } else {
+                    info!("invalid enum discriminant: {:#?}", val);
+                    crate::mir::interpret::struct_error(
+                        tcx.at(tcx.def_span(expr_did)),
+                        "constant evaluation of enum discriminant resulted in non-integer",
+                    )
+                    .emit();
+                    None
+                }
+            }
+            Err(err) => {
+                let msg = match err {
+                    ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => {
+                        "enum discriminant evaluation failed"
+                    }
+                    ErrorHandled::TooGeneric => "enum discriminant depends on generics",
+                };
+                tcx.sess.delay_span_bug(tcx.def_span(expr_did), msg);
+                None
+            }
+        }
+    }
+
+    #[inline]
+    pub fn discriminants(
+        &'tcx self,
+        tcx: TyCtxt<'tcx>,
+    ) -> impl Iterator<Item = (VariantIdx, Discr<'tcx>)> + Captures<'tcx> {
+        assert!(self.is_enum());
+        let repr_type = self.repr.discr_type();
+        let initial = repr_type.initial_discriminant(tcx);
+        let mut prev_discr = None::<Discr<'tcx>>;
+        self.variants.iter_enumerated().map(move |(i, v)| {
+            let mut discr = prev_discr.map_or(initial, |d| d.wrap_incr(tcx));
+            if let VariantDiscr::Explicit(expr_did) = v.discr {
+                if let Some(new_discr) = self.eval_explicit_discr(tcx, expr_did) {
+                    discr = new_discr;
+                }
+            }
+            prev_discr = Some(discr);
+
+            (i, discr)
+        })
+    }
+
+    #[inline]
+    pub fn variant_range(&self) -> Range<VariantIdx> {
+        VariantIdx::new(0)..VariantIdx::new(self.variants.len())
+    }
+
+    /// Computes the discriminant value used by a specific variant.
+    /// Unlike `discriminants`, this is (amortized) constant-time,
+    /// only doing at most one query for evaluating an explicit
+    /// discriminant (the last one before the requested variant),
+    /// assuming there are no constant-evaluation errors there.
+    #[inline]
+    pub fn discriminant_for_variant(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        variant_index: VariantIdx,
+    ) -> Discr<'tcx> {
+        assert!(self.is_enum());
+        let (val, offset) = self.discriminant_def_for_variant(variant_index);
+        let explicit_value = val
+            .and_then(|expr_did| self.eval_explicit_discr(tcx, expr_did))
+            .unwrap_or_else(|| self.repr.discr_type().initial_discriminant(tcx));
+        explicit_value.checked_add(tcx, offset as u128).0
+    }
+
+    /// Yields a `DefId` for the discriminant and an offset to add to it
+    /// Alternatively, if there is no explicit discriminant, returns the
+    /// inferred discriminant directly.
+    pub fn discriminant_def_for_variant(&self, variant_index: VariantIdx) -> (Option<DefId>, u32) {
+        assert!(!self.variants.is_empty());
+        let mut explicit_index = variant_index.as_u32();
+        let expr_did;
+        loop {
+            match self.variants[VariantIdx::from_u32(explicit_index)].discr {
+                ty::VariantDiscr::Relative(0) => {
+                    expr_did = None;
+                    break;
+                }
+                ty::VariantDiscr::Relative(distance) => {
+                    explicit_index -= distance;
+                }
+                ty::VariantDiscr::Explicit(did) => {
+                    expr_did = Some(did);
+                    break;
+                }
+            }
+        }
+        (expr_did, variant_index.as_u32() - explicit_index)
+    }
+
+    pub fn destructor(&self, tcx: TyCtxt<'tcx>) -> Option<Destructor> {
+        tcx.adt_destructor(self.did)
+    }
+
+    /// Returns a list of types such that `Self: Sized` if and only
+    /// if that type is `Sized`, or `TyErr` if this type is recursive.
+    ///
+    /// Oddly enough, checking that the sized-constraint is `Sized` is
+    /// actually more expressive than checking all members:
+    /// the `Sized` trait is inductive, so an associated type that references
+    /// `Self` would prevent its containing ADT from being `Sized`.
+    ///
+    /// Due to normalization being eager, this applies even if
+    /// the associated type is behind a pointer (e.g., issue #31299).
+    pub fn sized_constraint(&self, tcx: TyCtxt<'tcx>) -> &'tcx [Ty<'tcx>] {
+        tcx.adt_sized_constraint(self.did).0
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/assoc.rs b/compiler/rustc_middle/src/ty/assoc.rs
new file mode 100644
index 00000000000..d005f63ed43
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/assoc.rs
@@ -0,0 +1,170 @@
+pub use self::AssocItemContainer::*;
+
+use crate::ty;
+use rustc_data_structures::sorted_map::SortedIndexMultiMap;
+use rustc_hir as hir;
+use rustc_hir::def::{DefKind, Namespace};
+use rustc_hir::def_id::DefId;
+use rustc_span::symbol::{Ident, Symbol};
+
+use super::{TyCtxt, Visibility};
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, HashStable, Hash)]
+pub enum AssocItemContainer {
+    TraitContainer(DefId),
+    ImplContainer(DefId),
+}
+
+impl AssocItemContainer {
+    /// Asserts that this is the `DefId` of an associated item declared
+    /// in a trait, and returns the trait `DefId`.
+    pub fn assert_trait(&self) -> DefId {
+        match *self {
+            TraitContainer(id) => id,
+            _ => bug!("associated item has wrong container type: {:?}", self),
+        }
+    }
+
+    pub fn id(&self) -> DefId {
+        match *self {
+            TraitContainer(id) => id,
+            ImplContainer(id) => id,
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, HashStable, Eq, Hash)]
+pub struct AssocItem {
+    pub def_id: DefId,
+    #[stable_hasher(project(name))]
+    pub ident: Ident,
+    pub kind: AssocKind,
+    pub vis: Visibility,
+    pub defaultness: hir::Defaultness,
+    pub container: AssocItemContainer,
+
+    /// Whether this is a method with an explicit self
+    /// as its first parameter, allowing method calls.
+    pub fn_has_self_parameter: bool,
+}
+
+impl AssocItem {
+    pub fn signature(&self, tcx: TyCtxt<'_>) -> String {
+        match self.kind {
+            ty::AssocKind::Fn => {
+                // We skip the binder here because the binder would deanonymize all
+                // late-bound regions, and we don't want method signatures to show up
+                // `as for<'r> fn(&'r MyType)`.  Pretty-printing handles late-bound
+                // regions just fine, showing `fn(&MyType)`.
+                tcx.fn_sig(self.def_id).skip_binder().to_string()
+            }
+            ty::AssocKind::Type => format!("type {};", self.ident),
+            ty::AssocKind::Const => {
+                format!("const {}: {:?};", self.ident, tcx.type_of(self.def_id))
+            }
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Debug, HashStable, Eq, Hash)]
+pub enum AssocKind {
+    Const,
+    Fn,
+    Type,
+}
+
+impl AssocKind {
+    pub fn namespace(&self) -> Namespace {
+        match *self {
+            ty::AssocKind::Type => Namespace::TypeNS,
+            ty::AssocKind::Const | ty::AssocKind::Fn => Namespace::ValueNS,
+        }
+    }
+
+    pub fn as_def_kind(&self) -> DefKind {
+        match self {
+            AssocKind::Const => DefKind::AssocConst,
+            AssocKind::Fn => DefKind::AssocFn,
+            AssocKind::Type => DefKind::AssocTy,
+        }
+    }
+}
+
+/// A list of `ty::AssocItem`s in definition order that allows for efficient lookup by name.
+///
+/// When doing lookup by name, we try to postpone hygienic comparison for as long as possible since
+/// it is relatively expensive. Instead, items are indexed by `Symbol` and hygienic comparison is
+/// done only on items with the same name.
+#[derive(Debug, Clone, PartialEq, HashStable)]
+pub struct AssocItems<'tcx> {
+    pub(super) items: SortedIndexMultiMap<u32, Symbol, &'tcx ty::AssocItem>,
+}
+
+impl<'tcx> AssocItems<'tcx> {
+    /// Constructs an `AssociatedItems` map from a series of `ty::AssocItem`s in definition order.
+    pub fn new(items_in_def_order: impl IntoIterator<Item = &'tcx ty::AssocItem>) -> Self {
+        let items = items_in_def_order.into_iter().map(|item| (item.ident.name, item)).collect();
+        AssocItems { items }
+    }
+
+    /// Returns a slice of associated items in the order they were defined.
+    ///
+    /// New code should avoid relying on definition order. If you need a particular associated item
+    /// for a known trait, make that trait a lang item instead of indexing this array.
+    pub fn in_definition_order(&self) -> impl '_ + Iterator<Item = &ty::AssocItem> {
+        self.items.iter().map(|(_, v)| *v)
+    }
+
+    pub fn len(&self) -> usize {
+        self.items.len()
+    }
+
+    /// Returns an iterator over all associated items with the given name, ignoring hygiene.
+    pub fn filter_by_name_unhygienic(
+        &self,
+        name: Symbol,
+    ) -> impl '_ + Iterator<Item = &ty::AssocItem> {
+        self.items.get_by_key(&name).copied()
+    }
+
+    /// Returns an iterator over all associated items with the given name.
+    ///
+    /// Multiple items may have the same name if they are in different `Namespace`s. For example,
+    /// an associated type can have the same name as a method. Use one of the `find_by_name_and_*`
+    /// methods below if you know which item you are looking for.
+    pub fn filter_by_name(
+        &'a self,
+        tcx: TyCtxt<'a>,
+        ident: Ident,
+        parent_def_id: DefId,
+    ) -> impl 'a + Iterator<Item = &'a ty::AssocItem> {
+        self.filter_by_name_unhygienic(ident.name)
+            .filter(move |item| tcx.hygienic_eq(ident, item.ident, parent_def_id))
+    }
+
+    /// Returns the associated item with the given name and `AssocKind`, if one exists.
+    pub fn find_by_name_and_kind(
+        &self,
+        tcx: TyCtxt<'_>,
+        ident: Ident,
+        kind: AssocKind,
+        parent_def_id: DefId,
+    ) -> Option<&ty::AssocItem> {
+        self.filter_by_name_unhygienic(ident.name)
+            .filter(|item| item.kind == kind)
+            .find(|item| tcx.hygienic_eq(ident, item.ident, parent_def_id))
+    }
+
+    /// Returns the associated item with the given name in the given `Namespace`, if one exists.
+    pub fn find_by_name_and_namespace(
+        &self,
+        tcx: TyCtxt<'_>,
+        ident: Ident,
+        ns: Namespace,
+        parent_def_id: DefId,
+    ) -> Option<&ty::AssocItem> {
+        self.filter_by_name_unhygienic(ident.name)
+            .filter(|item| item.kind.namespace() == ns)
+            .find(|item| tcx.hygienic_eq(ident, item.ident, parent_def_id))
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
new file mode 100644
index 00000000000..887a5831cd7
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -0,0 +1,371 @@
+use crate::hir::place::{
+    Place as HirPlace, PlaceBase as HirPlaceBase, ProjectionKind as HirProjectionKind,
+};
+use crate::ty;
+
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
+use rustc_hir as hir;
+use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_span::Span;
+
+use super::{Ty, TyCtxt};
+
+use self::BorrowKind::*;
+
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    Hash,
+    TyEncodable,
+    TyDecodable,
+    TypeFoldable,
+    HashStable
+)]
+pub struct UpvarPath {
+    pub hir_id: hir::HirId,
+}
+
+/// Upvars do not get their own `NodeId`. Instead, we use the pair of
+/// the original var ID (that is, the root variable that is referenced
+/// by the upvar) and the ID of the closure expression.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
+pub struct UpvarId {
+    pub var_path: UpvarPath,
+    pub closure_expr_id: LocalDefId,
+}
+
+impl UpvarId {
+    pub fn new(var_hir_id: hir::HirId, closure_def_id: LocalDefId) -> UpvarId {
+        UpvarId { var_path: UpvarPath { hir_id: var_hir_id }, closure_expr_id: closure_def_id }
+    }
+}
+
+/// Information describing the capture of an upvar. This is computed
+/// during `typeck`, specifically by `regionck`.
+#[derive(PartialEq, Clone, Debug, Copy, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
+pub enum UpvarCapture<'tcx> {
+    /// Upvar is captured by value. This is always true when the
+    /// closure is labeled `move`, but can also be true in other cases
+    /// depending on inference.
+    ///
+    /// If the upvar was inferred to be captured by value (e.g. `move`
+    /// was not used), then the `Span` points to a usage that
+    /// required it. There may be more than one such usage
+    /// (e.g. `|| { a; a; }`), in which case we pick an
+    /// arbitrary one.
+    ByValue(Option<Span>),
+
+    /// Upvar is captured by reference.
+    ByRef(UpvarBorrow<'tcx>),
+}
+
+#[derive(PartialEq, Clone, Copy, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
+pub struct UpvarBorrow<'tcx> {
+    /// The kind of borrow: by-ref upvars have access to shared
+    /// immutable borrows, which are not part of the normal language
+    /// syntax.
+    pub kind: BorrowKind,
+
+    /// Region of the resulting reference.
+    pub region: ty::Region<'tcx>,
+}
+
+pub type UpvarListMap = FxHashMap<DefId, FxIndexMap<hir::HirId, UpvarId>>;
+pub type UpvarCaptureMap<'tcx> = FxHashMap<UpvarId, UpvarCapture<'tcx>>;
+
+/// Given the closure DefId this map provides a map of root variables to minimum
+/// set of `CapturedPlace`s that need to be tracked to support all captures of that closure.
+pub type MinCaptureInformationMap<'tcx> = FxHashMap<DefId, RootVariableMinCaptureList<'tcx>>;
+
+/// Part of `MinCaptureInformationMap`; Maps a root variable to the list of `CapturedPlace`.
+/// Used to track the minimum set of `Place`s that need to be captured to support all
+/// Places captured by the closure starting at a given root variable.
+///
+/// This provides a convenient and quick way of checking if a variable being used within
+/// a closure is a capture of a local variable.
+pub type RootVariableMinCaptureList<'tcx> = FxIndexMap<hir::HirId, MinCaptureList<'tcx>>;
+
+/// Part of `MinCaptureInformationMap`; List of `CapturePlace`s.
+pub type MinCaptureList<'tcx> = Vec<CapturedPlace<'tcx>>;
+
+/// Represents the various closure traits in the language. This
+/// will determine the type of the environment (`self`, in the
+/// desugaring) argument that the closure expects.
+///
+/// You can get the environment type of a closure using
+/// `tcx.closure_env_ty()`.
+#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
+#[derive(HashStable)]
+pub enum ClosureKind {
+    // Warning: Ordering is significant here! The ordering is chosen
+    // because the trait Fn is a subtrait of FnMut and so in turn, and
+    // hence we order it so that Fn < FnMut < FnOnce.
+    Fn,
+    FnMut,
+    FnOnce,
+}
+
+impl<'tcx> ClosureKind {
+    // This is the initial value used when doing upvar inference.
+    pub const LATTICE_BOTTOM: ClosureKind = ClosureKind::Fn;
+
+    /// Returns `true` if a type that impls this closure kind
+    /// must also implement `other`.
+    pub fn extends(self, other: ty::ClosureKind) -> bool {
+        matches!(
+            (self, other),
+            (ClosureKind::Fn, ClosureKind::Fn)
+                | (ClosureKind::Fn, ClosureKind::FnMut)
+                | (ClosureKind::Fn, ClosureKind::FnOnce)
+                | (ClosureKind::FnMut, ClosureKind::FnMut)
+                | (ClosureKind::FnMut, ClosureKind::FnOnce)
+                | (ClosureKind::FnOnce, ClosureKind::FnOnce)
+        )
+    }
+
+    /// Returns the representative scalar type for this closure kind.
+    /// See `TyS::to_opt_closure_kind` for more details.
+    pub fn to_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
+        match self {
+            ty::ClosureKind::Fn => tcx.types.i8,
+            ty::ClosureKind::FnMut => tcx.types.i16,
+            ty::ClosureKind::FnOnce => tcx.types.i32,
+        }
+    }
+}
+
+/// A composite describing a `Place` that is captured by a closure.
+#[derive(PartialEq, Clone, Debug, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
+pub struct CapturedPlace<'tcx> {
+    /// The `Place` that is captured.
+    pub place: HirPlace<'tcx>,
+
+    /// `CaptureKind` and expression(s) that resulted in such capture of `place`.
+    pub info: CaptureInfo<'tcx>,
+
+    /// Represents if `place` can be mutated or not.
+    pub mutability: hir::Mutability,
+}
+
+impl CapturedPlace<'tcx> {
+    /// Returns the hir-id of the root variable for the captured place.
+    /// e.g., if `a.b.c` was captured, would return the hir-id for `a`.
+    pub fn get_root_variable(&self) -> hir::HirId {
+        match self.place.base {
+            HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
+            base => bug!("Expected upvar, found={:?}", base),
+        }
+    }
+
+    /// Returns the `LocalDefId` of the closure that captured this Place
+    pub fn get_closure_local_def_id(&self) -> LocalDefId {
+        match self.place.base {
+            HirPlaceBase::Upvar(upvar_id) => upvar_id.closure_expr_id,
+            base => bug!("expected upvar, found={:?}", base),
+        }
+    }
+
+    /// Return span pointing to use that resulted in selecting the current capture kind
+    pub fn get_capture_kind_span(&self, tcx: TyCtxt<'tcx>) -> Span {
+        if let Some(capture_kind_expr_id) = self.info.capture_kind_expr_id {
+            tcx.hir().span(capture_kind_expr_id)
+        } else if let Some(path_expr_id) = self.info.path_expr_id {
+            tcx.hir().span(path_expr_id)
+        } else {
+            // Fallback on upvars mentioned if neither path or capture expr id is captured
+
+            // Safe to unwrap since we know this place is captured by the closure, therefore the closure must have upvars.
+            tcx.upvars_mentioned(self.get_closure_local_def_id()).unwrap()
+                [&self.get_root_variable()]
+                .span
+        }
+    }
+}
+
+/// Return true if the `proj_possible_ancestor` represents an ancestor path
+/// to `proj_capture` or `proj_possible_ancestor` is same as `proj_capture`,
+/// assuming they both start off of the same root variable.
+///
+/// **Note:** It's the caller's responsibility to ensure that both lists of projections
+///           start off of the same root variable.
+///
+/// Eg: 1. `foo.x` which is represented using `projections=[Field(x)]` is an ancestor of
+///        `foo.x.y` which is represented using `projections=[Field(x), Field(y)]`.
+///        Note both `foo.x` and `foo.x.y` start off of the same root variable `foo`.
+///     2. Since we only look at the projections here function will return `bar.x` as an a valid
+///        ancestor of `foo.x.y`. It's the caller's responsibility to ensure that both projections
+///        list are being applied to the same root variable.
+pub fn is_ancestor_or_same_capture(
+    proj_possible_ancestor: &[HirProjectionKind],
+    proj_capture: &[HirProjectionKind],
+) -> bool {
+    // We want to make sure `is_ancestor_or_same_capture("x.0.0", "x.0")` to return false.
+    // Therefore we can't just check if all projections are same in the zipped iterator below.
+    if proj_possible_ancestor.len() > proj_capture.len() {
+        return false;
+    }
+
+    proj_possible_ancestor.iter().zip(proj_capture).all(|(a, b)| a == b)
+}
+
+/// Part of `MinCaptureInformationMap`; describes the capture kind (&, &mut, move)
+/// for a particular capture as well as identifying the part of the source code
+/// that triggered this capture to occur.
+#[derive(PartialEq, Clone, Debug, Copy, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
+pub struct CaptureInfo<'tcx> {
+    /// Expr Id pointing to use that resulted in selecting the current capture kind
+    ///
+    /// Eg:
+    /// ```rust,no_run
+    /// let mut t = (0,1);
+    ///
+    /// let c = || {
+    ///     println!("{}",t); // L1
+    ///     t.1 = 4; // L2
+    /// };
+    /// ```
+    /// `capture_kind_expr_id` will point to the use on L2 and `path_expr_id` will point to the
+    /// use on L1.
+    ///
+    /// If the user doesn't enable feature `capture_disjoint_fields` (RFC 2229) then, it is
+    /// possible that we don't see the use of a particular place resulting in capture_kind_expr_id being
+    /// None. In such case we fallback on uvpars_mentioned for span.
+    ///
+    /// Eg:
+    /// ```rust,no_run
+    /// let x = 5;
+    ///
+    /// let c = || {
+    ///     let _ = x
+    /// };
+    /// ```
+    ///
+    /// In this example, if `capture_disjoint_fields` is **not** set, then x will be captured,
+    /// but we won't see it being used during capture analysis, since it's essentially a discard.
+    pub capture_kind_expr_id: Option<hir::HirId>,
+    /// Expr Id pointing to use that resulted the corresponding place being captured
+    ///
+    /// See `capture_kind_expr_id` for example.
+    ///
+    pub path_expr_id: Option<hir::HirId>,
+
+    /// Capture mode that was selected
+    pub capture_kind: UpvarCapture<'tcx>,
+}
+
+pub fn place_to_string_for_capture(tcx: TyCtxt<'tcx>, place: &HirPlace<'tcx>) -> String {
+    let name = match place.base {
+        HirPlaceBase::Upvar(upvar_id) => tcx.hir().name(upvar_id.var_path.hir_id).to_string(),
+        _ => bug!("Capture_information should only contain upvars"),
+    };
+    let mut curr_string = name;
+
+    for (i, proj) in place.projections.iter().enumerate() {
+        match proj.kind {
+            HirProjectionKind::Deref => {
+                curr_string = format!("*{}", curr_string);
+            }
+            HirProjectionKind::Field(idx, variant) => match place.ty_before_projection(i).kind() {
+                ty::Adt(def, ..) => {
+                    curr_string = format!(
+                        "{}.{}",
+                        curr_string,
+                        def.variants[variant].fields[idx as usize].ident.name.as_str()
+                    );
+                }
+                ty::Tuple(_) => {
+                    curr_string = format!("{}.{}", curr_string, idx);
+                }
+                _ => {
+                    bug!(
+                        "Field projection applied to a type other than Adt or Tuple: {:?}.",
+                        place.ty_before_projection(i).kind()
+                    )
+                }
+            },
+            proj => bug!("{:?} unexpected because it isn't captured", proj),
+        }
+    }
+
+    curr_string.to_string()
+}
+
+#[derive(Clone, PartialEq, Debug, TyEncodable, TyDecodable, TypeFoldable, Copy, HashStable)]
+pub enum BorrowKind {
+    /// Data must be immutable and is aliasable.
+    ImmBorrow,
+
+    /// Data must be immutable but not aliasable. This kind of borrow
+    /// cannot currently be expressed by the user and is used only in
+    /// implicit closure bindings. It is needed when the closure
+    /// is borrowing or mutating a mutable referent, e.g.:
+    ///
+    /// ```
+    /// let x: &mut isize = ...;
+    /// let y = || *x += 5;
+    /// ```
+    ///
+    /// If we were to try to translate this closure into a more explicit
+    /// form, we'd encounter an error with the code as written:
+    ///
+    /// ```
+    /// struct Env { x: & &mut isize }
+    /// let x: &mut isize = ...;
+    /// let y = (&mut Env { &x }, fn_ptr);  // Closure is pair of env and fn
+    /// fn fn_ptr(env: &mut Env) { **env.x += 5; }
+    /// ```
+    ///
+    /// This is then illegal because you cannot mutate a `&mut` found
+    /// in an aliasable location. To solve, you'd have to translate with
+    /// an `&mut` borrow:
+    ///
+    /// ```
+    /// struct Env { x: & &mut isize }
+    /// let x: &mut isize = ...;
+    /// let y = (&mut Env { &mut x }, fn_ptr); // changed from &x to &mut x
+    /// fn fn_ptr(env: &mut Env) { **env.x += 5; }
+    /// ```
+    ///
+    /// Now the assignment to `**env.x` is legal, but creating a
+    /// mutable pointer to `x` is not because `x` is not mutable. We
+    /// could fix this by declaring `x` as `let mut x`. This is ok in
+    /// user code, if awkward, but extra weird for closures, since the
+    /// borrow is hidden.
+    ///
+    /// So we introduce a "unique imm" borrow -- the referent is
+    /// immutable, but not aliasable. This solves the problem. For
+    /// simplicity, we don't give users the way to express this
+    /// borrow, it's just used when translating closures.
+    UniqueImmBorrow,
+
+    /// Data is mutable and not aliasable.
+    MutBorrow,
+}
+
+impl BorrowKind {
+    pub fn from_mutbl(m: hir::Mutability) -> BorrowKind {
+        match m {
+            hir::Mutability::Mut => MutBorrow,
+            hir::Mutability::Not => ImmBorrow,
+        }
+    }
+
+    /// Returns a mutability `m` such that an `&m T` pointer could be used to obtain this borrow
+    /// kind. Because borrow kinds are richer than mutabilities, we sometimes have to pick a
+    /// mutability that is stronger than necessary so that it at least *would permit* the borrow in
+    /// question.
+    pub fn to_mutbl_lossy(self) -> hir::Mutability {
+        match self {
+            MutBorrow => hir::Mutability::Mut,
+            ImmBorrow => hir::Mutability::Not,
+
+            // We have no type corresponding to a unique imm borrow, so
+            // use `&mut`. It gives all the capabilities of an `&uniq`
+            // and hence is a safe "over approximation".
+            UniqueImmBorrow => hir::Mutability::Mut,
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 73ad87a9ef2..d7767dc39cb 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -120,8 +120,9 @@ impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for Ty<'tcx> {
     }
 }
 
-impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for ty::Binder<ty::PredicateKind<'tcx>> {
+impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for ty::Binder<'tcx, ty::PredicateKind<'tcx>> {
     fn encode(&self, e: &mut E) -> Result<(), E::Error> {
+        self.bound_vars().encode(e)?;
         encode_with_shorthand(e, &self.skip_binder(), TyEncoder::predicate_shorthands)
     }
 }
@@ -188,7 +189,7 @@ pub trait TyDecoder<'tcx>: Decoder {
 }
 
 #[inline]
-pub fn decode_arena_allocable<'tcx, D, T: ArenaAllocatable<'tcx> + Decodable<D>>(
+fn decode_arena_allocable<'tcx, D, T: ArenaAllocatable<'tcx> + Decodable<D>>(
     decoder: &mut D,
 ) -> Result<&'tcx T, D::Error>
 where
@@ -198,7 +199,7 @@ where
 }
 
 #[inline]
-pub fn decode_arena_allocable_slice<'tcx, D, T: ArenaAllocatable<'tcx> + Decodable<D>>(
+fn decode_arena_allocable_slice<'tcx, D, T: ArenaAllocatable<'tcx> + Decodable<D>>(
     decoder: &mut D,
 ) -> Result<&'tcx [T], D::Error>
 where
@@ -226,18 +227,22 @@ impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for Ty<'tcx> {
     }
 }
 
-impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for ty::Binder<ty::PredicateKind<'tcx>> {
-    fn decode(decoder: &mut D) -> Result<ty::Binder<ty::PredicateKind<'tcx>>, D::Error> {
+impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for ty::Binder<'tcx, ty::PredicateKind<'tcx>> {
+    fn decode(decoder: &mut D) -> Result<ty::Binder<'tcx, ty::PredicateKind<'tcx>>, D::Error> {
+        let bound_vars = Decodable::decode(decoder)?;
         // Handle shorthands first, if we have an usize > 0x80.
-        Ok(ty::Binder::bind(if decoder.positioned_at_shorthand() {
-            let pos = decoder.read_usize()?;
-            assert!(pos >= SHORTHAND_OFFSET);
-            let shorthand = pos - SHORTHAND_OFFSET;
-
-            decoder.with_position(shorthand, ty::PredicateKind::decode)?
-        } else {
-            ty::PredicateKind::decode(decoder)?
-        }))
+        Ok(ty::Binder::bind_with_vars(
+            if decoder.positioned_at_shorthand() {
+                let pos = decoder.read_usize()?;
+                assert!(pos >= SHORTHAND_OFFSET);
+                let shorthand = pos - SHORTHAND_OFFSET;
+
+                decoder.with_position(shorthand, ty::PredicateKind::decode)?
+            } else {
+                ty::PredicateKind::decode(decoder)?
+            },
+            bound_vars,
+        ))
     }
 }
 
@@ -319,7 +324,7 @@ impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for ty::List<Ty<'tcx>> {
 }
 
 impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D>
-    for ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>
+    for ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>
 {
     fn decode(decoder: &mut D) -> Result<&'tcx Self, D::Error> {
         let len = decoder.read_usize()?;
@@ -333,6 +338,16 @@ impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for ty::Const<'tcx> {
     }
 }
 
+impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for [ty::ValTree<'tcx>] {
+    fn decode(decoder: &mut D) -> Result<&'tcx Self, D::Error> {
+        Ok(decoder.tcx().arena.alloc_from_iter(
+            (0..decoder.read_usize()?)
+                .map(|_| Decodable::decode(decoder))
+                .collect::<Result<Vec<_>, _>>()?,
+        ))
+    }
+}
+
 impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for Allocation {
     fn decode(decoder: &mut D) -> Result<&'tcx Self, D::Error> {
         Ok(decoder.tcx().intern_const_alloc(Decodable::decode(decoder)?))
@@ -369,15 +384,23 @@ impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for [mir::abstract_const::N
     }
 }
 
+impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for ty::List<ty::BoundVariableKind> {
+    fn decode(decoder: &mut D) -> Result<&'tcx Self, D::Error> {
+        let len = decoder.read_usize()?;
+        Ok(decoder.tcx().mk_bound_variable_kinds((0..len).map(|_| Decodable::decode(decoder)))?)
+    }
+}
+
 impl_decodable_via_ref! {
     &'tcx ty::TypeckResults<'tcx>,
     &'tcx ty::List<Ty<'tcx>>,
-    &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+    &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     &'tcx Allocation,
     &'tcx mir::Body<'tcx>,
     &'tcx mir::UnsafetyCheckResult,
     &'tcx mir::BorrowCheckResult<'tcx>,
-    &'tcx mir::coverage::CodeRegion
+    &'tcx mir::coverage::CodeRegion,
+    &'tcx ty::List<ty::BoundVariableKind>
 }
 
 #[macro_export]
@@ -462,6 +485,11 @@ macro_rules! implement_ty_decoder {
                     read_str -> Cow<'_, str>;
                 }
 
+                #[inline]
+                fn read_raw_bytes_into(&mut self, bytes: &mut [u8]) -> Result<(), Self::Error> {
+                    self.opaque.read_raw_bytes_into(bytes)
+                }
+
                 fn error(&mut self, err: &str) -> Self::Error {
                     self.opaque.error(err)
                 }
@@ -473,14 +501,16 @@ macro_rules! implement_ty_decoder {
 macro_rules! impl_binder_encode_decode {
     ($($t:ty),+ $(,)?) => {
         $(
-            impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for ty::Binder<$t> {
+            impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for ty::Binder<'tcx, $t> {
                 fn encode(&self, e: &mut E) -> Result<(), E::Error> {
+                    self.bound_vars().encode(e)?;
                     self.as_ref().skip_binder().encode(e)
                 }
             }
-            impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for ty::Binder<$t> {
+            impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for ty::Binder<'tcx, $t> {
                 fn decode(decoder: &mut D) -> Result<Self, D::Error> {
-                    Ok(ty::Binder::bind(Decodable::decode(decoder)?))
+                    let bound_vars = Decodable::decode(decoder)?;
+                    Ok(ty::Binder::bind_with_vars(Decodable::decode(decoder)?, bound_vars))
                 }
             }
         )*
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index ed953b98113..c78151271c1 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -5,14 +5,16 @@ use crate::ty::{self, Ty, TyCtxt};
 use crate::ty::{ParamEnv, ParamEnvAnd};
 use rustc_errors::ErrorReported;
 use rustc_hir as hir;
-use rustc_hir::def_id::LocalDefId;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_macros::HashStable;
 
 mod int;
 mod kind;
+mod valtree;
 
 pub use int::*;
 pub use kind::*;
+pub use valtree::*;
 
 /// Typed constant value.
 #[derive(Copy, Clone, Debug, Hash, TyEncodable, TyDecodable, Eq, PartialEq, Ord, PartialOrd)]
@@ -23,7 +25,7 @@ pub struct Const<'tcx> {
     pub val: ConstKind<'tcx>,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(Const<'_>, 48);
 
 impl<'tcx> Const<'tcx> {
@@ -96,18 +98,18 @@ impl<'tcx> Const<'tcx> {
                 let name = tcx.hir().name(hir_id);
                 ty::ConstKind::Param(ty::ParamConst::new(index, name))
             }
-            _ => ty::ConstKind::Unevaluated(
-                def.to_global(),
-                InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
-                None,
-            ),
+            _ => ty::ConstKind::Unevaluated(ty::Unevaluated {
+                def: def.to_global(),
+                substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
+                promoted: None,
+            }),
         };
 
         tcx.mk_const(ty::Const { val, ty })
     }
 
-    #[inline]
     /// Interns the given value as a constant.
+    #[inline]
     pub fn from_value(tcx: TyCtxt<'tcx>, val: ConstValue<'tcx>, ty: Ty<'tcx>) -> &'tcx Self {
         tcx.mk_const(Self { val: ConstKind::Value(val), ty })
     }
@@ -200,3 +202,18 @@ impl<'tcx> Const<'tcx> {
             .unwrap_or_else(|| bug!("expected usize, got {:#?}", self))
     }
 }
+
+pub fn const_param_default<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx Const<'tcx> {
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+    let default_def_id = match tcx.hir().get(hir_id) {
+        hir::Node::GenericParam(hir::GenericParam {
+            kind: hir::GenericParamKind::Const { ty: _, default: Some(ac) },
+            ..
+        }) => tcx.hir().local_def_id(ac.hir_id),
+        _ => span_bug!(
+            tcx.def_span(def_id),
+            "`const_param_default` expected a generic parameter with a constant"
+        ),
+    };
+    Const::from_anon_const(tcx, default_def_id)
+}
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index 63e95f25bb7..8ed8ea6a0bc 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -5,6 +5,8 @@ use rustc_target::abi::{Size, TargetDataLayout};
 use std::convert::{TryFrom, TryInto};
 use std::fmt;
 
+use crate::ty::TyCtxt;
+
 #[derive(Copy, Clone)]
 /// A type for representing any integer. Only used for printing.
 pub struct ConstInt {
@@ -239,6 +241,11 @@ impl ScalarInt {
             Err(self.size())
         }
     }
+
+    #[inline]
+    pub fn try_to_machine_usize(&self, tcx: TyCtxt<'tcx>) -> Result<u64, Size> {
+        Ok(self.to_bits(tcx.data_layout.pointer_size)? as u64)
+    }
 }
 
 macro_rules! from {
@@ -277,6 +284,18 @@ macro_rules! try_from {
 from!(u8, u16, u32, u64, u128, bool);
 try_from!(u8, u16, u32, u64, u128);
 
+impl TryFrom<ScalarInt> for bool {
+    type Error = Size;
+    #[inline]
+    fn try_from(int: ScalarInt) -> Result<Self, Size> {
+        int.to_bits(Size::from_bytes(1)).and_then(|u| match u {
+            0 => Ok(false),
+            1 => Ok(true),
+            _ => Err(Size::from_bytes(1)),
+        })
+    }
+}
+
 impl From<char> for ScalarInt {
     #[inline]
     fn from(c: char) -> Self {
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index a2638d8bdda..875d8d00a93 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -1,3 +1,5 @@
+use std::convert::TryInto;
+
 use crate::mir::interpret::ConstValue;
 use crate::mir::interpret::Scalar;
 use crate::mir::Promoted;
@@ -9,9 +11,19 @@ use rustc_hir::def_id::DefId;
 use rustc_macros::HashStable;
 use rustc_target::abi::Size;
 
+use super::ScalarInt;
+/// An unevaluated, potentially generic, constant.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
+#[derive(Hash, HashStable)]
+pub struct Unevaluated<'tcx> {
+    pub def: ty::WithOptConstParam<DefId>,
+    pub substs: SubstsRef<'tcx>,
+    pub promoted: Option<Promoted>,
+}
+
 /// Represents a constant in Rust.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
-#[derive(HashStable)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
+#[derive(Hash, HashStable)]
 pub enum ConstKind<'tcx> {
     /// A const generic parameter.
     Param(ty::ParamConst),
@@ -27,7 +39,7 @@ pub enum ConstKind<'tcx> {
 
     /// Used in the HIR by using `Unevaluated` everywhere and later normalizing to one of the other
     /// variants when the code is monomorphic enough for that.
-    Unevaluated(ty::WithOptConstParam<DefId>, SubstsRef<'tcx>, Option<Promoted>),
+    Unevaluated(Unevaluated<'tcx>),
 
     /// Used to hold computed value.
     Value(ConstValue<'tcx>),
@@ -37,7 +49,7 @@ pub enum ConstKind<'tcx> {
     Error(ty::DelaySpanBugEmitted),
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(ConstKind<'_>, 40);
 
 impl<'tcx> ConstKind<'tcx> {
@@ -52,13 +64,18 @@ impl<'tcx> ConstKind<'tcx> {
     }
 
     #[inline]
+    pub fn try_to_scalar_int(self) -> Option<ScalarInt> {
+        Some(self.try_to_value()?.try_to_scalar()?.assert_int())
+    }
+
+    #[inline]
     pub fn try_to_bits(self, size: Size) -> Option<u128> {
-        self.try_to_value()?.try_to_bits(size)
+        self.try_to_scalar_int()?.to_bits(size).ok()
     }
 
     #[inline]
     pub fn try_to_bool(self) -> Option<bool> {
-        self.try_to_value()?.try_to_bool()
+        self.try_to_scalar_int()?.try_into().ok()
     }
 
     #[inline]
@@ -93,7 +110,7 @@ impl<'tcx> ConstKind<'tcx> {
         tcx: TyCtxt<'tcx>,
         param_env: ParamEnv<'tcx>,
     ) -> Option<Result<ConstValue<'tcx>, ErrorReported>> {
-        if let ConstKind::Unevaluated(def, substs, promoted) = self {
+        if let ConstKind::Unevaluated(Unevaluated { def, substs, promoted }) = self {
             use crate::mir::interpret::ErrorHandled;
 
             // HACK(eddyb) this erases lifetimes even though `const_eval_resolve`
@@ -123,7 +140,8 @@ impl<'tcx> ConstKind<'tcx> {
             let (param_env, substs) = param_env_and_substs.into_parts();
             // try to resolve e.g. associated constants to their definition on an impl, and then
             // evaluate the const.
-            match tcx.const_eval_resolve(param_env, def, substs, promoted, None) {
+            match tcx.const_eval_resolve(param_env, ty::Unevaluated { def, substs, promoted }, None)
+            {
                 // NOTE(eddyb) `val` contains no lifetimes/types/consts,
                 // and we use the original type, so nothing from `substs`
                 // (which may be identity substs, see above),
diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs
new file mode 100644
index 00000000000..f1b78c87632
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/consts/valtree.rs
@@ -0,0 +1,34 @@
+use super::ScalarInt;
+use rustc_macros::HashStable;
+
+#[derive(Copy, Clone, Debug, Hash, TyEncodable, TyDecodable, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(HashStable)]
+/// This datastructure is used to represent the value of constants used in the type system.
+///
+/// We explicitly choose a different datastructure from the way values are processed within
+/// CTFE, as in the type system equal values (according to their `PartialEq`) must also have
+/// equal representation (`==` on the rustc data structure, e.g. `ValTree`) and vice versa.
+/// Since CTFE uses `AllocId` to represent pointers, it often happens that two different
+/// `AllocId`s point to equal values. So we may end up with different representations for
+/// two constants whose value is `&42`. Furthermore any kind of struct that has padding will
+/// have arbitrary values within that padding, even if the values of the struct are the same.
+///
+/// `ValTree` does not have this problem with representation, as it only contains integers or
+/// lists of (nested) `ValTree`.
+pub enum ValTree<'tcx> {
+    /// ZSTs, integers, `bool`, `char` are represented as scalars.
+    /// See the `ScalarInt` documentation for how `ScalarInt` guarantees that equal values
+    /// of these types have the same representation.
+    Leaf(ScalarInt),
+    /// The fields of any kind of aggregate. Structs, tuples and arrays are represented by
+    /// listing their fields' values in order.
+    /// Enums are represented by storing their discriminant as a field, followed by all
+    /// the fields of the variant.
+    Branch(&'tcx [ValTree<'tcx>]),
+}
+
+impl ValTree<'tcx> {
+    pub fn zst() -> Self {
+        Self::Branch(&[])
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index e1d79248171..bb2b00cbaea 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -30,9 +30,7 @@ use rustc_attr as attr;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
-use rustc_data_structures::stable_hasher::{
-    hash_stable_hashmap, HashStable, StableHasher, StableVec,
-};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableVec};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{self, Lock, Lrc, WorkerLocal};
 use rustc_errors::ErrorReported;
@@ -47,6 +45,7 @@ use rustc_hir::{
 };
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_macros::HashStable;
+use rustc_middle::mir::FakeReadCause;
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_session::config::{BorrowckMode, CrateType, OutputFilenames};
 use rustc_session::lint::{Level, Lint};
@@ -88,12 +87,16 @@ pub struct CtxtInterners<'tcx> {
     substs: InternedSet<'tcx, InternalSubsts<'tcx>>,
     canonical_var_infos: InternedSet<'tcx, List<CanonicalVarInfo<'tcx>>>,
     region: InternedSet<'tcx, RegionKind>,
-    poly_existential_predicates: InternedSet<'tcx, List<ty::Binder<ExistentialPredicate<'tcx>>>>,
+    poly_existential_predicates:
+        InternedSet<'tcx, List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>>,
     predicate: InternedSet<'tcx, PredicateInner<'tcx>>,
     predicates: InternedSet<'tcx, List<Predicate<'tcx>>>,
     projs: InternedSet<'tcx, List<ProjectionKind>>,
     place_elems: InternedSet<'tcx, List<PlaceElem<'tcx>>>,
     const_: InternedSet<'tcx, Const<'tcx>>,
+    /// Const allocations.
+    allocation: InternedSet<'tcx, Allocation>,
+    bound_variable_kinds: InternedSet<'tcx, List<ty::BoundVariableKind>>,
 }
 
 impl<'tcx> CtxtInterners<'tcx> {
@@ -111,6 +114,8 @@ impl<'tcx> CtxtInterners<'tcx> {
             projs: Default::default(),
             place_elems: Default::default(),
             const_: Default::default(),
+            allocation: Default::default(),
+            bound_variable_kinds: Default::default(),
         }
     }
 
@@ -134,7 +139,10 @@ impl<'tcx> CtxtInterners<'tcx> {
     }
 
     #[inline(never)]
-    fn intern_predicate(&self, kind: Binder<PredicateKind<'tcx>>) -> &'tcx PredicateInner<'tcx> {
+    fn intern_predicate(
+        &self,
+        kind: Binder<'tcx, PredicateKind<'tcx>>,
+    ) -> &'tcx PredicateInner<'tcx> {
         self.predicate
             .intern(kind, |kind| {
                 let flags = super::flags::FlagComputation::for_predicate(kind);
@@ -382,9 +390,6 @@ pub struct TypeckResults<'tcx> {
     /// <https://github.com/rust-lang/rfcs/blob/master/text/2005-match-ergonomics.md#definitions>
     pat_adjustments: ItemLocalMap<Vec<Ty<'tcx>>>,
 
-    /// Borrows
-    pub upvar_capture_map: ty::UpvarCaptureMap<'tcx>,
-
     /// Records the reasons that we picked the kind of each closure;
     /// not all closures are present in the map.
     closure_kind_origins: ItemLocalMap<(Span, HirPlace<'tcx>)>,
@@ -420,19 +425,37 @@ pub struct TypeckResults<'tcx> {
     /// by this function.
     pub concrete_opaque_types: FxHashMap<DefId, ResolvedOpaqueTy<'tcx>>,
 
-    /// Given the closure ID this map provides the list of UpvarIDs used by it.
-    /// The upvarID contains the HIR node ID and it also contains the full path
-    /// leading to the member of the struct or tuple that is used instead of the
-    /// entire variable.
-    pub closure_captures: ty::UpvarListMap,
-
     /// Tracks the minimum captures required for a closure;
     /// see `MinCaptureInformationMap` for more details.
     pub closure_min_captures: ty::MinCaptureInformationMap<'tcx>,
 
+    /// Tracks the fake reads required for a closure and the reason for the fake read.
+    /// When performing pattern matching for closures, there are times we don't end up
+    /// reading places that are mentioned in a closure (because of _ patterns). However,
+    /// to ensure the places are initialized, we introduce fake reads.
+    /// Consider these two examples:
+    /// ``` (discriminant matching with only wildcard arm)
+    /// let x: u8;
+    /// let c = || match x { _ => () };
+    /// ```
+    /// In this example, we don't need to actually read/borrow `x` in `c`, and so we don't
+    /// want to capture it. However, we do still want an error here, because `x` should have
+    /// to be initialized at the point where c is created. Therefore, we add a "fake read"
+    /// instead.
+    /// ``` (destructured assignments)
+    /// let c = || {
+    ///     let (t1, t2) = t;
+    /// }
+    /// ```
+    /// In the second example, we capture the disjoint fields of `t` (`t.0` & `t.1`), but
+    /// we never capture `t`. This becomes an issue when we build MIR as we require
+    /// information on `t` in order to create place `t.0` and `t.1`. We can solve this
+    /// issue by fake reading `t`.
+    pub closure_fake_reads: FxHashMap<DefId, Vec<(HirPlace<'tcx>, FakeReadCause, hir::HirId)>>,
+
     /// Stores the type, expression, span and optional scope span of all types
     /// that are live across the yield of this generator (if a generator).
-    pub generator_interior_types: ty::Binder<Vec<GeneratorInteriorTypeCause<'tcx>>>,
+    pub generator_interior_types: ty::Binder<'tcx, Vec<GeneratorInteriorTypeCause<'tcx>>>,
 
     /// We sometimes treat byte string literals (which are of type `&[u8; N]`)
     /// as `&[u8]`, depending on the pattern  in which they are used.
@@ -454,7 +477,6 @@ impl<'tcx> TypeckResults<'tcx> {
             adjustments: Default::default(),
             pat_binding_modes: Default::default(),
             pat_adjustments: Default::default(),
-            upvar_capture_map: Default::default(),
             closure_kind_origins: Default::default(),
             liberated_fn_sigs: Default::default(),
             fru_field_types: Default::default(),
@@ -462,8 +484,8 @@ impl<'tcx> TypeckResults<'tcx> {
             used_trait_imports: Lrc::new(Default::default()),
             tainted_by_errors: None,
             concrete_opaque_types: Default::default(),
-            closure_captures: Default::default(),
             closure_min_captures: Default::default(),
+            closure_fake_reads: Default::default(),
             generator_interior_types: ty::Binder::dummy(Default::default()),
             treat_byte_string_as_slice: Default::default(),
         }
@@ -646,10 +668,6 @@ impl<'tcx> TypeckResults<'tcx> {
             .flatten()
     }
 
-    pub fn upvar_capture(&self, upvar_id: ty::UpvarId) -> ty::UpvarCapture<'tcx> {
-        self.upvar_capture_map[&upvar_id]
-    }
-
     pub fn closure_kind_origins(&self) -> LocalTableInContext<'_, (Span, HirPlace<'tcx>)> {
         LocalTableInContext { hir_owner: self.hir_owner, data: &self.closure_kind_origins }
     }
@@ -703,23 +721,22 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for TypeckResults<'tcx> {
             ref adjustments,
             ref pat_binding_modes,
             ref pat_adjustments,
-            ref upvar_capture_map,
             ref closure_kind_origins,
             ref liberated_fn_sigs,
             ref fru_field_types,
-
             ref coercion_casts,
-
             ref used_trait_imports,
             tainted_by_errors,
             ref concrete_opaque_types,
-            ref closure_captures,
             ref closure_min_captures,
+            ref closure_fake_reads,
             ref generator_interior_types,
             ref treat_byte_string_as_slice,
         } = *self;
 
         hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| {
+            hcx.local_def_path_hash(hir_owner);
+
             type_dependent_defs.hash_stable(hcx, hasher);
             field_indices.hash_stable(hcx, hasher);
             user_provided_types.hash_stable(hcx, hasher);
@@ -729,17 +746,6 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for TypeckResults<'tcx> {
             adjustments.hash_stable(hcx, hasher);
             pat_binding_modes.hash_stable(hcx, hasher);
             pat_adjustments.hash_stable(hcx, hasher);
-            hash_stable_hashmap(hcx, hasher, upvar_capture_map, |up_var_id, hcx| {
-                let ty::UpvarId { var_path, closure_expr_id } = *up_var_id;
-
-                assert_eq!(var_path.hir_id.owner, hir_owner);
-
-                (
-                    hcx.local_def_path_hash(var_path.hir_id.owner),
-                    var_path.hir_id.local_id,
-                    hcx.local_def_path_hash(closure_expr_id),
-                )
-            });
 
             closure_kind_origins.hash_stable(hcx, hasher);
             liberated_fn_sigs.hash_stable(hcx, hasher);
@@ -748,8 +754,8 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for TypeckResults<'tcx> {
             used_trait_imports.hash_stable(hcx, hasher);
             tainted_by_errors.hash_stable(hcx, hasher);
             concrete_opaque_types.hash_stable(hcx, hasher);
-            closure_captures.hash_stable(hcx, hasher);
             closure_min_captures.hash_stable(hcx, hasher);
+            closure_fake_reads.hash_stable(hcx, hasher);
             generator_interior_types.hash_stable(hcx, hasher);
             treat_byte_string_as_slice.hash_stable(hcx, hasher);
         })
@@ -789,7 +795,7 @@ impl CanonicalUserType<'tcx> {
                     return false;
                 }
 
-                user_substs.substs.iter().zip(BoundVar::new(0)..).all(|(kind, cvar)| {
+                iter::zip(user_substs.substs, BoundVar::new(0)..).all(|(kind, cvar)| {
                     match kind.unpack() {
                         GenericArgKind::Type(ty) => match ty.kind() {
                             ty::Bound(debruijn, b) => {
@@ -804,7 +810,7 @@ impl CanonicalUserType<'tcx> {
                             ty::ReLateBound(debruijn, br) => {
                                 // We only allow a `ty::INNERMOST` index in substitutions.
                                 assert_eq!(*debruijn, ty::INNERMOST);
-                                cvar == br.assert_bound_var()
+                                cvar == br.var
                             }
                             _ => false,
                         },
@@ -1013,9 +1019,6 @@ pub struct GlobalCtxt<'tcx> {
     /// `#[rustc_const_stable]` and `#[rustc_const_unstable]` attributes
     const_stability_interner: ShardedHashMap<&'tcx attr::ConstStability, ()>,
 
-    /// Stores the value of constants (and deduplicates the actual memory)
-    allocation_interner: ShardedHashMap<&'tcx Allocation, ()>,
-
     /// Stores memory for globals (statics/consts).
     pub(crate) alloc_map: Lock<interpret::AllocMap<'tcx>>,
 
@@ -1058,7 +1061,10 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn intern_const_alloc(self, alloc: Allocation) -> &'tcx Allocation {
-        self.allocation_interner.intern(alloc, |alloc| self.arena.alloc(alloc))
+        self.interners
+            .allocation
+            .intern(alloc, |alloc| Interned(self.interners.arena.alloc(alloc)))
+            .0
     }
 
     /// Allocates a read-only byte or string literal for `mir::interpret`.
@@ -1091,13 +1097,16 @@ impl<'tcx> TyCtxt<'tcx> {
                 None => return Bound::Unbounded,
             };
             debug!("layout_scalar_valid_range: attr={:?}", attr);
-            for meta in attr.meta_item_list().expect("rustc_layout_scalar_valid_range takes args") {
-                match meta.literal().expect("attribute takes lit").kind {
-                    ast::LitKind::Int(a, _) => return Bound::Included(a),
-                    _ => span_bug!(attr.span, "rustc_layout_scalar_valid_range expects int arg"),
-                }
+            if let Some(
+                &[ast::NestedMetaItem::Literal(ast::Lit { kind: ast::LitKind::Int(a, _), .. })],
+            ) = attr.meta_item_list().as_deref()
+            {
+                Bound::Included(a)
+            } else {
+                self.sess
+                    .delay_span_bug(attr.span, "invalid rustc_layout_scalar_valid_range attribute");
+                Bound::Unbounded
             }
-            span_bug!(attr.span, "no arguments to `rustc_layout_scalar_valid_range` attribute");
         };
         (
             get(sym::rustc_layout_scalar_valid_range_start),
@@ -1174,7 +1183,6 @@ impl<'tcx> TyCtxt<'tcx> {
             layout_interner: Default::default(),
             stability_interner: Default::default(),
             const_stability_interner: Default::default(),
-            allocation_interner: Default::default(),
             alloc_map: Lock::new(interpret::AllocMap::new()),
             output_filenames: Arc::new(output_filenames.clone()),
         }
@@ -1610,13 +1618,15 @@ macro_rules! nop_list_lift {
 nop_lift! {type_; Ty<'a> => Ty<'tcx>}
 nop_lift! {region; Region<'a> => Region<'tcx>}
 nop_lift! {const_; &'a Const<'a> => &'tcx Const<'tcx>}
+nop_lift! {allocation; &'a Allocation => &'tcx Allocation}
 nop_lift! {predicate; &'a PredicateInner<'a> => &'tcx PredicateInner<'tcx>}
 
 nop_list_lift! {type_list; Ty<'a> => Ty<'tcx>}
-nop_list_lift! {poly_existential_predicates; ty::Binder<ExistentialPredicate<'a>> => ty::Binder<ExistentialPredicate<'tcx>>}
+nop_list_lift! {poly_existential_predicates; ty::Binder<'a, ExistentialPredicate<'a>> => ty::Binder<'tcx, ExistentialPredicate<'tcx>>}
 nop_list_lift! {predicates; Predicate<'a> => Predicate<'tcx>}
 nop_list_lift! {canonical_var_infos; CanonicalVarInfo<'a> => CanonicalVarInfo<'tcx>}
 nop_list_lift! {projs; ProjectionKind => ProjectionKind}
+nop_list_lift! {bound_variable_kinds; ty::BoundVariableKind => ty::BoundVariableKind}
 
 // This is the impl for `&'a InternalSubsts<'a>`.
 nop_list_lift! {substs; GenericArg<'a> => GenericArg<'tcx>}
@@ -1900,7 +1910,7 @@ impl<'tcx> TyCtxt<'tcx> {
                     "Const Stability interner: #{}",
                     self.0.const_stability_interner.len()
                 )?;
-                writeln!(fmt, "Allocation interner: #{}", self.0.allocation_interner.len())?;
+                writeln!(fmt, "Allocation interner: #{}", self.0.interners.allocation.len())?;
                 writeln!(fmt, "Layout interner: #{}", self.0.layout_interner.len())?;
 
                 Ok(())
@@ -1962,8 +1972,8 @@ impl<'tcx> Hash for Interned<'tcx, PredicateInner<'tcx>> {
     }
 }
 
-impl<'tcx> Borrow<Binder<PredicateKind<'tcx>>> for Interned<'tcx, PredicateInner<'tcx>> {
-    fn borrow<'a>(&'a self) -> &'a Binder<PredicateKind<'tcx>> {
+impl<'tcx> Borrow<Binder<'tcx, PredicateKind<'tcx>>> for Interned<'tcx, PredicateInner<'tcx>> {
+    fn borrow<'a>(&'a self) -> &'a Binder<'tcx, PredicateKind<'tcx>> {
         &self.0.kind
     }
 }
@@ -2001,6 +2011,26 @@ impl<'tcx> Borrow<Const<'tcx>> for Interned<'tcx, Const<'tcx>> {
     }
 }
 
+impl<'tcx> Borrow<Allocation> for Interned<'tcx, Allocation> {
+    fn borrow<'a>(&'a self) -> &'a Allocation {
+        &self.0
+    }
+}
+
+impl<'tcx> PartialEq for Interned<'tcx, Allocation> {
+    fn eq(&self, other: &Self) -> bool {
+        self.0 == other.0
+    }
+}
+
+impl<'tcx> Eq for Interned<'tcx, Allocation> {}
+
+impl<'tcx> Hash for Interned<'tcx, Allocation> {
+    fn hash<H: Hasher>(&self, s: &mut H) {
+        self.0.hash(s)
+    }
+}
+
 macro_rules! direct_interners {
     ($($name:ident: $method:ident($ty:ty),)+) => {
         $(impl<'tcx> PartialEq for Interned<'tcx, $ty> {
@@ -2049,10 +2079,11 @@ slice_interners!(
     substs: _intern_substs(GenericArg<'tcx>),
     canonical_var_infos: _intern_canonical_var_infos(CanonicalVarInfo<'tcx>),
     poly_existential_predicates:
-        _intern_poly_existential_predicates(ty::Binder<ExistentialPredicate<'tcx>>),
+        _intern_poly_existential_predicates(ty::Binder<'tcx, ExistentialPredicate<'tcx>>),
     predicates: _intern_predicates(Predicate<'tcx>),
     projs: _intern_projs(ProjectionKind),
     place_elems: _intern_place_elems(PlaceElem<'tcx>),
+    bound_variable_kinds: _intern_bound_variable_kinds(ty::BoundVariableKind),
 );
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -2135,7 +2166,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline]
-    pub fn mk_predicate(self, binder: Binder<PredicateKind<'tcx>>) -> Predicate<'tcx> {
+    pub fn mk_predicate(self, binder: Binder<'tcx, PredicateKind<'tcx>>) -> Predicate<'tcx> {
         let inner = self.interners.intern_predicate(binder);
         Predicate { inner }
     }
@@ -2144,7 +2175,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn reuse_or_mk_predicate(
         self,
         pred: Predicate<'tcx>,
-        binder: Binder<PredicateKind<'tcx>>,
+        binder: Binder<'tcx, PredicateKind<'tcx>>,
     ) -> Predicate<'tcx> {
         if pred.kind() != binder { self.mk_predicate(binder) } else { pred }
     }
@@ -2198,7 +2229,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let adt_def = self.adt_def(wrapper_def_id);
         let substs =
             InternalSubsts::for_item(self, wrapper_def_id, |param, substs| match param.kind {
-                GenericParamDefKind::Lifetime | GenericParamDefKind::Const => bug!(),
+                GenericParamDefKind::Lifetime | GenericParamDefKind::Const { .. } => bug!(),
                 GenericParamDefKind::Type { has_default, .. } => {
                     if param.index == 0 {
                         ty_param.into()
@@ -2266,11 +2297,6 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline]
-    pub fn mk_nil_ptr(self) -> Ty<'tcx> {
-        self.mk_imm_ptr(self.mk_unit())
-    }
-
-    #[inline]
     pub fn mk_array(self, ty: Ty<'tcx>, n: u64) -> Ty<'tcx> {
         self.mk_ty(Array(ty, ty::Const::from_usize(self, n)))
     }
@@ -2316,7 +2342,7 @@ impl<'tcx> TyCtxt<'tcx> {
     #[inline]
     pub fn mk_dynamic(
         self,
-        obj: &'tcx List<ty::Binder<ExistentialPredicate<'tcx>>>,
+        obj: &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>,
         reg: ty::Region<'tcx>,
     ) -> Ty<'tcx> {
         self.mk_ty(Dynamic(obj, reg))
@@ -2343,7 +2369,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline]
-    pub fn mk_generator_witness(self, types: ty::Binder<&'tcx List<Ty<'tcx>>>) -> Ty<'tcx> {
+    pub fn mk_generator_witness(self, types: ty::Binder<'tcx, &'tcx List<Ty<'tcx>>>) -> Ty<'tcx> {
         self.mk_ty(GeneratorWitness(types))
     }
 
@@ -2393,7 +2419,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 self.mk_region(ty::ReEarlyBound(param.to_early_bound_region_data())).into()
             }
             GenericParamDefKind::Type { .. } => self.mk_ty_param(param.index, param.name).into(),
-            GenericParamDefKind::Const => {
+            GenericParamDefKind::Const { .. } => {
                 self.mk_const_param(param.index, param.name, self.type_of(param.def_id)).into()
             }
         }
@@ -2448,8 +2474,8 @@ impl<'tcx> TyCtxt<'tcx> {
 
     pub fn intern_poly_existential_predicates(
         self,
-        eps: &[ty::Binder<ExistentialPredicate<'tcx>>],
-    ) -> &'tcx List<ty::Binder<ExistentialPredicate<'tcx>>> {
+        eps: &[ty::Binder<'tcx, ExistentialPredicate<'tcx>>],
+    ) -> &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>> {
         assert!(!eps.is_empty());
         assert!(
             eps.array_windows()
@@ -2494,6 +2520,13 @@ impl<'tcx> TyCtxt<'tcx> {
         if ts.is_empty() { List::empty() } else { self._intern_canonical_var_infos(ts) }
     }
 
+    pub fn intern_bound_variable_kinds(
+        self,
+        ts: &[ty::BoundVariableKind],
+    ) -> &'tcx List<ty::BoundVariableKind> {
+        if ts.is_empty() { List::empty() } else { self._intern_bound_variable_kinds(ts) }
+    }
+
     pub fn mk_fn_sig<I>(
         self,
         inputs: I,
@@ -2515,8 +2548,8 @@ impl<'tcx> TyCtxt<'tcx> {
 
     pub fn mk_poly_existential_predicates<
         I: InternAs<
-            [ty::Binder<ExistentialPredicate<'tcx>>],
-            &'tcx List<ty::Binder<ExistentialPredicate<'tcx>>>,
+            [ty::Binder<'tcx, ExistentialPredicate<'tcx>>],
+            &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>,
         >,
     >(
         self,
@@ -2554,6 +2587,15 @@ impl<'tcx> TyCtxt<'tcx> {
         self.mk_substs(iter::once(self_ty.into()).chain(rest.iter().cloned()))
     }
 
+    pub fn mk_bound_variable_kinds<
+        I: InternAs<[ty::BoundVariableKind], &'tcx List<ty::BoundVariableKind>>,
+    >(
+        self,
+        iter: I,
+    ) -> I::Output {
+        iter.intern_with(|xs| self.intern_bound_variable_kinds(xs))
+    }
+
     /// Walks upwards from `id` to find a node which might change lint levels with attributes.
     /// It stops at `bound` and just returns it if reached.
     pub fn maybe_lint_level_root_bounded(self, mut id: HirId, bound: HirId) -> HirId {
@@ -2618,6 +2660,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn named_region(self, id: HirId) -> Option<resolve_lifetime::Region> {
+        debug!(?id, "named_region");
         self.named_region_map(id.owner).and_then(|map| map.get(&id.local_id).cloned())
     }
 
@@ -2626,9 +2669,19 @@ impl<'tcx> TyCtxt<'tcx> {
             .map_or(false, |(owner, set)| owner == id.owner && set.contains(&id.local_id))
     }
 
-    pub fn object_lifetime_defaults(self, id: HirId) -> Option<&'tcx [ObjectLifetimeDefault]> {
+    pub fn object_lifetime_defaults(self, id: HirId) -> Option<Vec<ObjectLifetimeDefault>> {
         self.object_lifetime_defaults_map(id.owner)
-            .and_then(|map| map.get(&id.local_id).map(|v| &**v))
+    }
+
+    pub fn late_bound_vars(self, id: HirId) -> &'tcx List<ty::BoundVariableKind> {
+        self.mk_bound_variable_kinds(
+            self.late_bound_vars_map(id.owner)
+                .and_then(|map| map.get(&id.local_id).cloned())
+                .unwrap_or_else(|| {
+                    bug!("No bound vars found for {:?} ({:?})", self.hir().node_to_string(id), id)
+                })
+                .iter(),
+        )
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index f41bb7e6d63..982c8a354b4 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -314,6 +314,7 @@ impl<'v> hir::intravisit::Visitor<'v> for TraitObjectVisitor<'v> {
                         hir::LifetimeName::ImplicitObjectLifetimeDefault | hir::LifetimeName::Static,
                     ..
                 },
+                _,
             ) => {
                 self.0.push(ty);
             }
diff --git a/compiler/rustc_middle/src/ty/erase_regions.rs b/compiler/rustc_middle/src/ty/erase_regions.rs
index 4412ba9408c..759d1a017aa 100644
--- a/compiler/rustc_middle/src/ty/erase_regions.rs
+++ b/compiler/rustc_middle/src/ty/erase_regions.rs
@@ -1,3 +1,4 @@
+use crate::mir;
 use crate::ty::fold::{TypeFoldable, TypeFolder};
 use crate::ty::{self, Ty, TyCtxt, TypeFlags};
 
@@ -43,7 +44,7 @@ impl TypeFolder<'tcx> for RegionEraserVisitor<'tcx> {
         if ty.needs_infer() { ty.super_fold_with(self) } else { self.tcx.erase_regions_ty(ty) }
     }
 
-    fn fold_binder<T>(&mut self, t: ty::Binder<T>) -> ty::Binder<T>
+    fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
     where
         T: TypeFoldable<'tcx>,
     {
@@ -65,4 +66,8 @@ impl TypeFolder<'tcx> for RegionEraserVisitor<'tcx> {
             _ => self.tcx.lifetimes.re_erased,
         }
     }
+
+    fn fold_mir_const(&mut self, c: mir::ConstantKind<'tcx>) -> mir::ConstantKind<'tcx> {
+        c.super_fold_with(self)
+    }
 }
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index bf315c81588..008e6d015e8 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -12,7 +12,6 @@ use rustc_target::spec::abi;
 
 use std::borrow::Cow;
 use std::fmt;
-use std::ops::Deref;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable)]
 pub struct ExpectedFound<T> {
@@ -37,6 +36,7 @@ pub enum TypeError<'tcx> {
     UnsafetyMismatch(ExpectedFound<hir::Unsafety>),
     AbiMismatch(ExpectedFound<abi::Abi>),
     Mutability,
+    ArgumentMutability(usize),
     TupleSize(ExpectedFound<usize>),
     FixedArraySize(ExpectedFound<u64>),
     ArgCount,
@@ -47,6 +47,7 @@ pub enum TypeError<'tcx> {
     RegionsPlaceholderMismatch,
 
     Sorts(ExpectedFound<Ty<'tcx>>),
+    ArgumentSorts(ExpectedFound<Ty<'tcx>>, usize),
     IntMismatch(ExpectedFound<ty::IntVarValue>),
     FloatMismatch(ExpectedFound<ty::FloatTy>),
     Traits(ExpectedFound<DefId>),
@@ -58,7 +59,9 @@ pub enum TypeError<'tcx> {
     CyclicTy(Ty<'tcx>),
     CyclicConst(&'tcx ty::Const<'tcx>),
     ProjectionMismatched(ExpectedFound<DefId>),
-    ExistentialMismatch(ExpectedFound<&'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>>),
+    ExistentialMismatch(
+        ExpectedFound<&'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>>,
+    ),
     ObjectUnsafeCoercion(DefId),
     ConstMismatch(ExpectedFound<&'tcx ty::Const<'tcx>>),
 
@@ -109,7 +112,7 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
             AbiMismatch(values) => {
                 write!(f, "expected {} fn, found {} fn", values.expected, values.found)
             }
-            Mutability => write!(f, "types differ in mutability"),
+            ArgumentMutability(_) | Mutability => write!(f, "types differ in mutability"),
             TupleSize(values) => write!(
                 f,
                 "expected a tuple with {} element{}, \
@@ -141,7 +144,7 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
                 br_string(br)
             ),
             RegionsPlaceholderMismatch => write!(f, "one type is more general than the other"),
-            Sorts(values) => ty::tls::with(|tcx| {
+            ArgumentSorts(values, _) | Sorts(values) => ty::tls::with(|tcx| {
                 report_maybe_different(
                     f,
                     &values.expected.sort_string(tcx),
@@ -198,10 +201,11 @@ impl<'tcx> TypeError<'tcx> {
         use self::TypeError::*;
         match self {
             CyclicTy(_) | CyclicConst(_) | UnsafetyMismatch(_) | Mismatch | AbiMismatch(_)
-            | FixedArraySize(_) | Sorts(_) | IntMismatch(_) | FloatMismatch(_)
-            | VariadicMismatch(_) | TargetFeatureCast(_) => false,
+            | FixedArraySize(_) | ArgumentSorts(..) | Sorts(_) | IntMismatch(_)
+            | FloatMismatch(_) | VariadicMismatch(_) | TargetFeatureCast(_) => false,
 
             Mutability
+            | ArgumentMutability(_)
             | TupleSize(_)
             | ArgCount
             | RegionsDoesNotOutlive(..)
@@ -338,7 +342,7 @@ impl<'tcx> TyCtxt<'tcx> {
         use self::TypeError::*;
         debug!("note_and_explain_type_err err={:?} cause={:?}", err, cause);
         match err {
-            Sorts(values) => {
+            ArgumentSorts(values, _) | Sorts(values) => {
                 match (values.expected.kind(), values.found.kind()) {
                     (ty::Closure(..), ty::Closure(..)) => {
                         db.note("no two closures, even if identical, have the same type");
@@ -510,13 +514,18 @@ impl<T> Trait<T> for X {
                             "consider constraining the associated type `{}` to `{}`",
                             values.found, values.expected,
                         );
-                        if !self.suggest_constraint(
+                        if !(self.suggest_constraining_opaque_associated_type(
+                            db,
+                            &msg,
+                            proj_ty,
+                            values.expected,
+                        ) || self.suggest_constraint(
                             db,
                             &msg,
                             body_owner_def_id,
                             proj_ty,
                             values.expected,
-                        ) {
+                        )) {
                             db.help(&msg);
                             db.note(
                                 "for more information, visit \
@@ -548,7 +557,6 @@ impl<T> Trait<T> for X {
             TargetFeatureCast(def_id) => {
                 let attrs = self.get_attrs(*def_id);
                 let target_spans = attrs
-                    .deref()
                     .iter()
                     .filter(|attr| attr.has_name(sym::target_feature))
                     .map(|attr| attr.span);
@@ -701,20 +709,7 @@ impl<T> Trait<T> for X {
             }
         }
 
-        if let ty::Opaque(def_id, _) = *proj_ty.self_ty().kind() {
-            // When the expected `impl Trait` is not defined in the current item, it will come from
-            // a return type. This can occur when dealing with `TryStream` (#71035).
-            if self.constrain_associated_type_structured_suggestion(
-                db,
-                self.def_span(def_id),
-                &assoc,
-                proj_ty.trait_ref_and_own_substs(self).1,
-                values.found,
-                &msg,
-            ) {
-                return;
-            }
-        }
+        self.suggest_constraining_opaque_associated_type(db, &msg, proj_ty, values.found);
 
         if self.point_at_associated_type(db, body_owner_def_id, values.found) {
             return;
@@ -752,6 +747,30 @@ fn foo(&self) -> Self::T { String::new() }
         }
     }
 
+    /// When the expected `impl Trait` is not defined in the current item, it will come from
+    /// a return type. This can occur when dealing with `TryStream` (#71035).
+    fn suggest_constraining_opaque_associated_type(
+        self,
+        db: &mut DiagnosticBuilder<'_>,
+        msg: &str,
+        proj_ty: &ty::ProjectionTy<'tcx>,
+        ty: Ty<'tcx>,
+    ) -> bool {
+        let assoc = self.associated_item(proj_ty.item_def_id);
+        if let ty::Opaque(def_id, _) = *proj_ty.self_ty().kind() {
+            self.constrain_associated_type_structured_suggestion(
+                db,
+                self.def_span(def_id),
+                &assoc,
+                proj_ty.trait_ref_and_own_substs(self).1,
+                ty,
+                &msg,
+            )
+        } else {
+            false
+        }
+    }
+
     fn point_at_methods_that_satisfy_associated_type(
         self,
         db: &mut DiagnosticBuilder<'_>,
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index 6ecd1ebf370..01bc5cc761c 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -22,7 +22,7 @@ impl FlagComputation {
         result
     }
 
-    pub fn for_predicate(binder: ty::Binder<ty::PredicateKind<'_>>) -> FlagComputation {
+    pub fn for_predicate(binder: ty::Binder<'tcx, ty::PredicateKind<'_>>) -> FlagComputation {
         let mut result = FlagComputation::new();
         result.add_predicate(binder);
         result
@@ -53,7 +53,7 @@ impl FlagComputation {
 
     /// Adds the flags/depth from a set of types that appear within the current type, but within a
     /// region binder.
-    fn bound_computation<T, F>(&mut self, value: ty::Binder<T>, f: F)
+    fn bound_computation<T, F>(&mut self, value: ty::Binder<'_, T>, f: F)
     where
         F: FnOnce(&mut Self, T),
     {
@@ -204,7 +204,7 @@ impl FlagComputation {
         }
     }
 
-    fn add_predicate(&mut self, binder: ty::Binder<ty::PredicateKind<'_>>) {
+    fn add_predicate(&mut self, binder: ty::Binder<'tcx, ty::PredicateKind<'_>>) {
         self.bound_computation(binder, |computation, atom| computation.add_predicate_atom(atom));
     }
 
@@ -270,10 +270,7 @@ impl FlagComputation {
     fn add_const(&mut self, c: &ty::Const<'_>) {
         self.add_ty(c.ty);
         match c.val {
-            ty::ConstKind::Unevaluated(_, substs, _) => {
-                self.add_substs(substs);
-                self.add_flags(TypeFlags::HAS_CT_PROJECTION);
-            }
+            ty::ConstKind::Unevaluated(unevaluated) => self.add_unevaluated_const(unevaluated),
             ty::ConstKind::Infer(infer) => {
                 self.add_flags(TypeFlags::STILL_FURTHER_SPECIALIZABLE);
                 match infer {
@@ -297,6 +294,11 @@ impl FlagComputation {
         }
     }
 
+    fn add_unevaluated_const(&mut self, ct: ty::Unevaluated<'tcx>) {
+        self.add_substs(ct.substs);
+        self.add_flags(TypeFlags::HAS_CT_PROJECTION);
+    }
+
     fn add_existential_projection(&mut self, projection: &ty::ExistentialProjection<'_>) {
         self.add_substs(projection.substs);
         self.add_ty(projection.ty);
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 4c7db4e803b..eb6d163312c 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -30,11 +30,13 @@
 //!
 //! These methods return true to indicate that the visitor has found what it is
 //! looking for, and does not need to visit anything else.
+use crate::mir;
 use crate::ty::{self, flags::FlagComputation, Binder, Ty, TyCtxt, TypeFlags};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 
 use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::sso::SsoHashSet;
 use std::collections::BTreeMap;
 use std::fmt;
 use std::ops::ControlFlow;
@@ -42,7 +44,7 @@ use std::ops::ControlFlow;
 /// This trait is implemented for every type that can be folded.
 /// Basically, every type that has a corresponding method in `TypeFolder`.
 ///
-/// To implement this conveniently, use the derive macro located in librustc_macros.
+/// To implement this conveniently, use the derive macro located in `rustc_macros`.
 pub trait TypeFoldable<'tcx>: fmt::Debug + Clone {
     fn super_fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self;
     fn fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self {
@@ -161,7 +163,7 @@ impl TypeFoldable<'tcx> for hir::Constness {
 pub trait TypeFolder<'tcx>: Sized {
     fn tcx<'a>(&'a self) -> TyCtxt<'tcx>;
 
-    fn fold_binder<T>(&mut self, t: Binder<T>) -> Binder<T>
+    fn fold_binder<T>(&mut self, t: Binder<'tcx, T>) -> Binder<'tcx, T>
     where
         T: TypeFoldable<'tcx>,
     {
@@ -179,12 +181,19 @@ pub trait TypeFolder<'tcx>: Sized {
     fn fold_const(&mut self, c: &'tcx ty::Const<'tcx>) -> &'tcx ty::Const<'tcx> {
         c.super_fold_with(self)
     }
+
+    fn fold_mir_const(&mut self, c: mir::ConstantKind<'tcx>) -> mir::ConstantKind<'tcx> {
+        bug!("most type folders should not be folding MIR datastructures: {:?}", c)
+    }
 }
 
 pub trait TypeVisitor<'tcx>: Sized {
     type BreakTy = !;
 
-    fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<Self::BreakTy> {
+    fn visit_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: &Binder<'tcx, T>,
+    ) -> ControlFlow<Self::BreakTy> {
         t.super_visit_with(self)
     }
 
@@ -322,7 +331,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
             fn visit_binder<T: TypeFoldable<'tcx>>(
                 &mut self,
-                t: &Binder<T>,
+                t: &Binder<'tcx, T>,
             ) -> ControlFlow<Self::BreakTy> {
                 self.outer_index.shift_in(1);
                 let result = t.as_ref().skip_binder().visit_with(self);
@@ -400,7 +409,10 @@ impl<'a, 'tcx> TypeFolder<'tcx> for RegionFolder<'a, 'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: ty::Binder<T>) -> ty::Binder<T> {
+    fn fold_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: ty::Binder<'tcx, T>,
+    ) -> ty::Binder<'tcx, T> {
         self.current_index.shift_in(1);
         let t = t.super_fold_with(self);
         self.current_index.shift_out(1);
@@ -439,18 +451,18 @@ struct BoundVarReplacer<'a, 'tcx> {
     /// the ones we have visited.
     current_index: ty::DebruijnIndex,
 
-    fld_r: &'a mut (dyn FnMut(ty::BoundRegion) -> ty::Region<'tcx> + 'a),
-    fld_t: &'a mut (dyn FnMut(ty::BoundTy) -> Ty<'tcx> + 'a),
-    fld_c: &'a mut (dyn FnMut(ty::BoundVar, Ty<'tcx>) -> &'tcx ty::Const<'tcx> + 'a),
+    fld_r: Option<&'a mut (dyn FnMut(ty::BoundRegion) -> ty::Region<'tcx> + 'a)>,
+    fld_t: Option<&'a mut (dyn FnMut(ty::BoundTy) -> Ty<'tcx> + 'a)>,
+    fld_c: Option<&'a mut (dyn FnMut(ty::BoundVar, Ty<'tcx>) -> &'tcx ty::Const<'tcx> + 'a)>,
 }
 
 impl<'a, 'tcx> BoundVarReplacer<'a, 'tcx> {
-    fn new<F, G, H>(tcx: TyCtxt<'tcx>, fld_r: &'a mut F, fld_t: &'a mut G, fld_c: &'a mut H) -> Self
-    where
-        F: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
-        G: FnMut(ty::BoundTy) -> Ty<'tcx>,
-        H: FnMut(ty::BoundVar, Ty<'tcx>) -> &'tcx ty::Const<'tcx>,
-    {
+    fn new(
+        tcx: TyCtxt<'tcx>,
+        fld_r: Option<&'a mut (dyn FnMut(ty::BoundRegion) -> ty::Region<'tcx> + 'a)>,
+        fld_t: Option<&'a mut (dyn FnMut(ty::BoundTy) -> Ty<'tcx> + 'a)>,
+        fld_c: Option<&'a mut (dyn FnMut(ty::BoundVar, Ty<'tcx>) -> &'tcx ty::Const<'tcx> + 'a)>,
+    ) -> Self {
         BoundVarReplacer { tcx, current_index: ty::INNERMOST, fld_r, fld_t, fld_c }
     }
 }
@@ -460,7 +472,10 @@ impl<'a, 'tcx> TypeFolder<'tcx> for BoundVarReplacer<'a, 'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: ty::Binder<T>) -> ty::Binder<T> {
+    fn fold_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: ty::Binder<'tcx, T>,
+    ) -> ty::Binder<'tcx, T> {
         self.current_index.shift_in(1);
         let t = t.super_fold_with(self);
         self.current_index.shift_out(1);
@@ -469,63 +484,58 @@ impl<'a, 'tcx> TypeFolder<'tcx> for BoundVarReplacer<'a, 'tcx> {
 
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         match *t.kind() {
-            ty::Bound(debruijn, bound_ty) => {
-                if debruijn == self.current_index {
-                    let fld_t = &mut self.fld_t;
+            ty::Bound(debruijn, bound_ty) if debruijn == self.current_index => {
+                if let Some(fld_t) = self.fld_t.as_mut() {
                     let ty = fld_t(bound_ty);
-                    ty::fold::shift_vars(self.tcx, &ty, self.current_index.as_u32())
-                } else {
-                    t
+                    return ty::fold::shift_vars(self.tcx, &ty, self.current_index.as_u32());
                 }
             }
-            _ => {
-                if !t.has_vars_bound_at_or_above(self.current_index) {
-                    // Nothing more to substitute.
-                    t
-                } else {
-                    t.super_fold_with(self)
-                }
+            _ if t.has_vars_bound_at_or_above(self.current_index) => {
+                return t.super_fold_with(self);
             }
+            _ => {}
         }
+        t
     }
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
             ty::ReLateBound(debruijn, br) if debruijn == self.current_index => {
-                let fld_r = &mut self.fld_r;
-                let region = fld_r(br);
-                if let ty::ReLateBound(debruijn1, br) = *region {
-                    // If the callback returns a late-bound region,
-                    // that region should always use the INNERMOST
-                    // debruijn index. Then we adjust it to the
-                    // correct depth.
-                    assert_eq!(debruijn1, ty::INNERMOST);
-                    self.tcx.mk_region(ty::ReLateBound(debruijn, br))
-                } else {
-                    region
+                if let Some(fld_r) = self.fld_r.as_mut() {
+                    let region = fld_r(br);
+                    return if let ty::ReLateBound(debruijn1, br) = *region {
+                        // If the callback returns a late-bound region,
+                        // that region should always use the INNERMOST
+                        // debruijn index. Then we adjust it to the
+                        // correct depth.
+                        assert_eq!(debruijn1, ty::INNERMOST);
+                        self.tcx.mk_region(ty::ReLateBound(debruijn, br))
+                    } else {
+                        region
+                    };
                 }
             }
-            _ => r,
+            _ => {}
         }
+        r
     }
 
     fn fold_const(&mut self, ct: &'tcx ty::Const<'tcx>) -> &'tcx ty::Const<'tcx> {
-        if let ty::Const { val: ty::ConstKind::Bound(debruijn, bound_const), ty } = *ct {
-            if debruijn == self.current_index {
-                let fld_c = &mut self.fld_c;
-                let ct = fld_c(bound_const, ty);
-                ty::fold::shift_vars(self.tcx, &ct, self.current_index.as_u32())
-            } else {
-                ct
+        match *ct {
+            ty::Const { val: ty::ConstKind::Bound(debruijn, bound_const), ty }
+                if debruijn == self.current_index =>
+            {
+                if let Some(fld_c) = self.fld_c.as_mut() {
+                    let ct = fld_c(bound_const, ty);
+                    return ty::fold::shift_vars(self.tcx, &ct, self.current_index.as_u32());
+                }
             }
-        } else {
-            if !ct.has_vars_bound_at_or_above(self.current_index) {
-                // Nothing more to substitute.
-                ct
-            } else {
-                ct.super_fold_with(self)
+            _ if ct.has_vars_bound_at_or_above(self.current_index) => {
+                return ct.super_fold_with(self);
             }
+            _ => {}
         }
+        ct
     }
 }
 
@@ -543,21 +553,23 @@ impl<'tcx> TyCtxt<'tcx> {
     /// contain escaping bound types.
     pub fn replace_late_bound_regions<T, F>(
         self,
-        value: Binder<T>,
+        value: Binder<'tcx, T>,
         mut fld_r: F,
     ) -> (T, BTreeMap<ty::BoundRegion, ty::Region<'tcx>>)
     where
         F: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
         T: TypeFoldable<'tcx>,
     {
-        // identity for bound types and consts
-        let fld_t = |bound_ty| self.mk_ty(ty::Bound(ty::INNERMOST, bound_ty));
-        let fld_c = |bound_ct, ty| {
-            self.mk_const(ty::Const { val: ty::ConstKind::Bound(ty::INNERMOST, bound_ct), ty })
-        };
         let mut region_map = BTreeMap::new();
-        let real_fld_r = |br: ty::BoundRegion| *region_map.entry(br).or_insert_with(|| fld_r(br));
-        let value = self.replace_escaping_bound_vars(value.skip_binder(), real_fld_r, fld_t, fld_c);
+        let mut real_fld_r =
+            |br: ty::BoundRegion| *region_map.entry(br).or_insert_with(|| fld_r(br));
+        let value = value.skip_binder();
+        let value = if !value.has_escaping_bound_vars() {
+            value
+        } else {
+            let mut replacer = BoundVarReplacer::new(self, Some(&mut real_fld_r), None, None);
+            value.fold_with(&mut replacer)
+        };
         (value, region_map)
     }
 
@@ -580,7 +592,8 @@ impl<'tcx> TyCtxt<'tcx> {
         if !value.has_escaping_bound_vars() {
             value
         } else {
-            let mut replacer = BoundVarReplacer::new(self, &mut fld_r, &mut fld_t, &mut fld_c);
+            let mut replacer =
+                BoundVarReplacer::new(self, Some(&mut fld_r), Some(&mut fld_t), Some(&mut fld_c));
             value.fold_with(&mut replacer)
         }
     }
@@ -590,7 +603,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// types.
     pub fn replace_bound_vars<T, F, G, H>(
         self,
-        value: Binder<T>,
+        value: Binder<'tcx, T>,
         mut fld_r: F,
         fld_t: G,
         fld_c: H,
@@ -609,7 +622,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Replaces any late-bound regions bound in `value` with
     /// free variants attached to `all_outlive_scope`.
-    pub fn liberate_late_bound_regions<T>(self, all_outlive_scope: DefId, value: ty::Binder<T>) -> T
+    pub fn liberate_late_bound_regions<T>(
+        self,
+        all_outlive_scope: DefId,
+        value: ty::Binder<'tcx, T>,
+    ) -> T
     where
         T: TypeFoldable<'tcx>,
     {
@@ -622,13 +639,49 @@ impl<'tcx> TyCtxt<'tcx> {
         .0
     }
 
+    pub fn shift_bound_var_indices<T>(self, bound_vars: usize, value: T) -> T
+    where
+        T: TypeFoldable<'tcx>,
+    {
+        self.replace_escaping_bound_vars(
+            value,
+            |r| {
+                self.mk_region(ty::ReLateBound(
+                    ty::INNERMOST,
+                    ty::BoundRegion {
+                        var: ty::BoundVar::from_usize(r.var.as_usize() + bound_vars),
+                        kind: r.kind,
+                    },
+                ))
+            },
+            |t| {
+                self.mk_ty(ty::Bound(
+                    ty::INNERMOST,
+                    ty::BoundTy {
+                        var: ty::BoundVar::from_usize(t.var.as_usize() + bound_vars),
+                        kind: t.kind,
+                    },
+                ))
+            },
+            |c, ty| {
+                self.mk_const(ty::Const {
+                    val: ty::ConstKind::Bound(
+                        ty::INNERMOST,
+                        ty::BoundVar::from_usize(c.as_usize() + bound_vars),
+                    ),
+                    ty,
+                })
+            },
+        )
+    }
+
     /// Returns a set of all late-bound regions that are constrained
     /// by `value`, meaning that if we instantiate those LBR with
     /// variables and equate `value` with something else, those
     /// variables will also be equated.
     pub fn collect_constrained_late_bound_regions<T>(
         self,
-        value: &Binder<T>,
+        value: &Binder<'tcx, T>,
     ) -> FxHashSet<ty::BoundRegionKind>
     where
         T: TypeFoldable<'tcx>,
@@ -639,7 +692,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Returns a set of all late-bound regions that appear in `value` anywhere.
     pub fn collect_referenced_late_bound_regions<T>(
         self,
-        value: &Binder<T>,
+        value: &Binder<'tcx, T>,
     ) -> FxHashSet<ty::BoundRegionKind>
     where
         T: TypeFoldable<'tcx>,
@@ -649,7 +702,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
     fn collect_late_bound_regions<T>(
         self,
-        value: &Binder<T>,
+        value: &Binder<'tcx, T>,
         just_constraint: bool,
     ) -> FxHashSet<ty::BoundRegionKind>
     where
@@ -663,7 +716,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Replaces any late-bound regions bound in `value` with `'erased`. Useful in codegen but also
     /// method lookup and a few other places where precise region relationships are not required.
-    pub fn erase_late_bound_regions<T>(self, value: Binder<T>) -> T
+    pub fn erase_late_bound_regions<T>(self, value: Binder<'tcx, T>) -> T
     where
         T: TypeFoldable<'tcx>,
     {
@@ -678,20 +731,205 @@ impl<'tcx> TyCtxt<'tcx> {
     /// `FnSig`s or `TraitRef`s which are equivalent up to region naming will become
     /// structurally identical. For example, `for<'a, 'b> fn(&'a isize, &'b isize)` and
     /// `for<'a, 'b> fn(&'b isize, &'a isize)` will become identical after anonymization.
-    pub fn anonymize_late_bound_regions<T>(self, sig: Binder<T>) -> Binder<T>
+    pub fn anonymize_late_bound_regions<T>(self, sig: Binder<'tcx, T>) -> Binder<'tcx, T>
     where
         T: TypeFoldable<'tcx>,
     {
         let mut counter = 0;
-        Binder::bind(
-            self.replace_late_bound_regions(sig, |_| {
-                let br = ty::BoundRegion { kind: ty::BrAnon(counter) };
+        let inner = self
+            .replace_late_bound_regions(sig, |_| {
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_u32(counter),
+                    kind: ty::BrAnon(counter),
+                };
                 let r = self.mk_region(ty::ReLateBound(ty::INNERMOST, br));
                 counter += 1;
                 r
             })
-            .0,
-        )
+            .0;
+        let bound_vars = self.mk_bound_variable_kinds(
+            (0..counter).map(|i| ty::BoundVariableKind::Region(ty::BrAnon(i))),
+        );
+        Binder::bind_with_vars(inner, bound_vars)
+    }
+}
+
+pub struct BoundVarsCollector<'tcx> {
+    binder_index: ty::DebruijnIndex,
+    vars: BTreeMap<u32, ty::BoundVariableKind>,
+    // We may encounter the same variable at different levels of binding, so
+    // this can't just be `Ty`
+    visited: SsoHashSet<(ty::DebruijnIndex, Ty<'tcx>)>,
+}
+
+impl<'tcx> BoundVarsCollector<'tcx> {
+    pub fn new() -> Self {
+        BoundVarsCollector {
+            binder_index: ty::INNERMOST,
+            vars: BTreeMap::new(),
+            visited: SsoHashSet::default(),
+        }
+    }
+
+    pub fn into_vars(self, tcx: TyCtxt<'tcx>) -> &'tcx ty::List<ty::BoundVariableKind> {
+        let max = self.vars.iter().map(|(k, _)| *k).max().unwrap_or_else(|| 0);
+        for i in 0..max {
+            if let None = self.vars.get(&i) {
+                panic!("Unknown variable: {:?}", i);
+            }
+        }
+
+        tcx.mk_bound_variable_kinds(self.vars.into_iter().map(|(_, v)| v))
+    }
+}
+
+impl<'tcx> TypeVisitor<'tcx> for BoundVarsCollector<'tcx> {
+    type BreakTy = ();
+
+    fn visit_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: &Binder<'tcx, T>,
+    ) -> ControlFlow<Self::BreakTy> {
+        self.binder_index.shift_in(1);
+        let result = t.super_visit_with(self);
+        self.binder_index.shift_out(1);
+        result
+    }
+
+    fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
+        if t.outer_exclusive_binder < self.binder_index
+            || !self.visited.insert((self.binder_index, t))
+        {
+            return ControlFlow::CONTINUE;
+        }
+        use std::collections::btree_map::Entry;
+        match *t.kind() {
+            ty::Bound(debruijn, bound_ty) if debruijn == self.binder_index => {
+                match self.vars.entry(bound_ty.var.as_u32()) {
+                    Entry::Vacant(entry) => {
+                        entry.insert(ty::BoundVariableKind::Ty(bound_ty.kind));
+                    }
+                    Entry::Occupied(entry) => match entry.get() {
+                        ty::BoundVariableKind::Ty(_) => {}
+                        _ => bug!("Conflicting bound vars"),
+                    },
+                }
+            }
+
+            _ => (),
+        };
+
+        t.super_visit_with(self)
+    }
+
+    fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
+        match r {
+            ty::ReLateBound(index, _br) if *index == self.binder_index => {
+                // If you hit this, you should be using `Binder::bind_with_vars` or `Binder::rebind`
+                bug!("Trying to collect bound vars with a bound region: {:?} {:?}", index, _br)
+            }
+
+            _ => (),
+        };
+
+        r.super_visit_with(self)
+    }
+}
+
+pub struct ValidateBoundVars<'tcx> {
+    bound_vars: &'tcx ty::List<ty::BoundVariableKind>,
+    binder_index: ty::DebruijnIndex,
+    // We may encounter the same variable at different levels of binding, so
+    // this can't just be `Ty`
+    visited: SsoHashSet<(ty::DebruijnIndex, Ty<'tcx>)>,
+}
+
+impl<'tcx> ValidateBoundVars<'tcx> {
+    pub fn new(bound_vars: &'tcx ty::List<ty::BoundVariableKind>) -> Self {
+        ValidateBoundVars {
+            bound_vars,
+            binder_index: ty::INNERMOST,
+            visited: SsoHashSet::default(),
+        }
+    }
+}
+
+impl<'tcx> TypeVisitor<'tcx> for ValidateBoundVars<'tcx> {
+    type BreakTy = ();
+
+    fn visit_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: &Binder<'tcx, T>,
+    ) -> ControlFlow<Self::BreakTy> {
+        self.binder_index.shift_in(1);
+        let result = t.super_visit_with(self);
+        self.binder_index.shift_out(1);
+        result
+    }
+
+    fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
+        if t.outer_exclusive_binder < self.binder_index
+            || !self.visited.insert((self.binder_index, t))
+        {
+            return ControlFlow::BREAK;
+        }
+        match *t.kind() {
+            ty::Bound(debruijn, bound_ty) if debruijn == self.binder_index => {
+                if self.bound_vars.len() <= bound_ty.var.as_usize() {
+                    bug!("Not enough bound vars: {:?} not found in {:?}", t, self.bound_vars);
+                }
+                let list_var = self.bound_vars[bound_ty.var.as_usize()];
+                match list_var {
+                    ty::BoundVariableKind::Ty(kind) => {
+                        if kind != bound_ty.kind {
+                            bug!(
+                                "Mismatched type kinds: {:?} doesn't var in list {:?}",
+                                bound_ty.kind,
+                                list_var
+                            );
+                        }
+                    }
+                    _ => {
+                        bug!("Mismatched bound variable kinds! Expected type, found {:?}", list_var)
+                    }
+                }
+            }
+
+            _ => (),
+        };
+
+        t.super_visit_with(self)
+    }
+
+    fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
+        match r {
+            ty::ReLateBound(index, br) if *index == self.binder_index => {
+                if self.bound_vars.len() <= br.var.as_usize() {
+                    bug!("Not enough bound vars: {:?} not found in {:?}", *br, self.bound_vars);
+                }
+                let list_var = self.bound_vars[br.var.as_usize()];
+                match list_var {
+                    ty::BoundVariableKind::Region(kind) => {
+                        if kind != br.kind {
+                            bug!(
+                                "Mismatched region kinds: {:?} doesn't match var ({:?}) in list ({:?})",
+                                br.kind,
+                                list_var,
+                                self.bound_vars
+                            );
+                        }
+                    }
+                    _ => bug!(
+                        "Mismatched bound variable kinds! Expected region, found {:?}",
+                        list_var
+                    ),
+                }
+            }
+
+            _ => (),
+        };
+
+        r.super_visit_with(self)
     }
 }
 
@@ -721,7 +959,10 @@ impl TypeFolder<'tcx> for Shifter<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: ty::Binder<T>) -> ty::Binder<T> {
+    fn fold_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: ty::Binder<'tcx, T>,
+    ) -> ty::Binder<'tcx, T> {
         self.current_index.shift_in(1);
         let t = t.super_fold_with(self);
         self.current_index.shift_out(1);
@@ -830,7 +1071,10 @@ struct HasEscapingVarsVisitor {
 impl<'tcx> TypeVisitor<'tcx> for HasEscapingVarsVisitor {
     type BreakTy = FoundEscapingVars;
 
-    fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<Self::BreakTy> {
+    fn visit_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: &Binder<'tcx, T>,
+    ) -> ControlFlow<Self::BreakTy> {
         self.outer_index.shift_in(1);
         let result = t.super_visit_with(self);
         self.outer_index.shift_out(1);
@@ -976,7 +1220,10 @@ impl LateBoundRegionsCollector {
 }
 
 impl<'tcx> TypeVisitor<'tcx> for LateBoundRegionsCollector {
-    fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<Self::BreakTy> {
+    fn visit_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: &Binder<'tcx, T>,
+    ) -> ControlFlow<Self::BreakTy> {
         self.current_index.shift_in(1);
         let result = t.super_visit_with(self);
         self.current_index.shift_out(1);
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
new file mode 100644
index 00000000000..d30a8693959
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -0,0 +1,261 @@
+use crate::middle::resolve_lifetime::ObjectLifetimeDefault;
+use crate::ty;
+use crate::ty::subst::{Subst, SubstsRef};
+use rustc_ast as ast;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir as hir;
+use rustc_hir::def_id::DefId;
+use rustc_span::symbol::Symbol;
+use rustc_span::Span;
+
+use super::{EarlyBoundRegion, InstantiatedPredicates, ParamConst, ParamTy, Predicate, TyCtxt};
+
+#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+pub enum GenericParamDefKind {
+    Lifetime,
+    Type {
+        has_default: bool,
+        object_lifetime_default: ObjectLifetimeDefault,
+        synthetic: Option<hir::SyntheticTyParamKind>,
+    },
+    Const {
+        has_default: bool,
+    },
+}
+
+impl GenericParamDefKind {
+    pub fn descr(&self) -> &'static str {
+        match self {
+            GenericParamDefKind::Lifetime => "lifetime",
+            GenericParamDefKind::Type { .. } => "type",
+            GenericParamDefKind::Const { .. } => "constant",
+        }
+    }
+    pub fn to_ord(&self, tcx: TyCtxt<'_>) -> ast::ParamKindOrd {
+        match self {
+            GenericParamDefKind::Lifetime => ast::ParamKindOrd::Lifetime,
+            GenericParamDefKind::Type { .. } => ast::ParamKindOrd::Type,
+            GenericParamDefKind::Const { .. } => {
+                ast::ParamKindOrd::Const { unordered: tcx.features().const_generics }
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+pub struct GenericParamDef {
+    pub name: Symbol,
+    pub def_id: DefId,
+    pub index: u32,
+
+    /// `pure_wrt_drop`, set by the (unsafe) `#[may_dangle]` attribute
+    /// on generic parameter `'a`/`T`, asserts data behind the parameter
+    /// `'a`/`T` won't be accessed during the parent type's `Drop` impl.
+    pub pure_wrt_drop: bool,
+
+    pub kind: GenericParamDefKind,
+}
+
+impl GenericParamDef {
+    pub fn to_early_bound_region_data(&self) -> ty::EarlyBoundRegion {
+        if let GenericParamDefKind::Lifetime = self.kind {
+            ty::EarlyBoundRegion { def_id: self.def_id, index: self.index, name: self.name }
+        } else {
+            bug!("cannot convert a non-lifetime parameter def to an early bound region")
+        }
+    }
+}
+
+#[derive(Default)]
+pub struct GenericParamCount {
+    pub lifetimes: usize,
+    pub types: usize,
+    pub consts: usize,
+}
+
+/// Information about the formal type/lifetime parameters associated
+/// with an item or method. Analogous to `hir::Generics`.
+///
+/// The ordering of parameters is the same as in `Subst` (excluding child generics):
+/// `Self` (optionally), `Lifetime` params..., `Type` params...
+#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+pub struct Generics {
+    pub parent: Option<DefId>,
+    pub parent_count: usize,
+    pub params: Vec<GenericParamDef>,
+
+    /// Reverse map to the `index` field of each `GenericParamDef`.
+    #[stable_hasher(ignore)]
+    pub param_def_id_to_index: FxHashMap<DefId, u32>,
+
+    pub has_self: bool,
+    pub has_late_bound_regions: Option<Span>,
+}
+
+impl<'tcx> Generics {
+    pub fn count(&self) -> usize {
+        self.parent_count + self.params.len()
+    }
+
+    pub fn own_counts(&self) -> GenericParamCount {
+        // We could cache this as a property of `GenericParamCount`, but
+        // the aim is to refactor this away entirely eventually and the
+        // presence of this method will be a constant reminder.
+        let mut own_counts = GenericParamCount::default();
+
+        for param in &self.params {
+            match param.kind {
+                GenericParamDefKind::Lifetime => own_counts.lifetimes += 1,
+                GenericParamDefKind::Type { .. } => own_counts.types += 1,
+                GenericParamDefKind::Const { .. } => own_counts.consts += 1,
+            }
+        }
+
+        own_counts
+    }
+
+    pub fn own_defaults(&self) -> GenericParamCount {
+        let mut own_defaults = GenericParamCount::default();
+
+        for param in &self.params {
+            match param.kind {
+                GenericParamDefKind::Lifetime => (),
+                GenericParamDefKind::Type { has_default, .. } => {
+                    own_defaults.types += has_default as usize;
+                }
+                GenericParamDefKind::Const { has_default } => {
+                    own_defaults.consts += has_default as usize;
+                }
+            }
+        }
+
+        own_defaults
+    }
+
+    pub fn requires_monomorphization(&self, tcx: TyCtxt<'tcx>) -> bool {
+        if self.own_requires_monomorphization() {
+            return true;
+        }
+
+        if let Some(parent_def_id) = self.parent {
+            let parent = tcx.generics_of(parent_def_id);
+            parent.requires_monomorphization(tcx)
+        } else {
+            false
+        }
+    }
+
+    pub fn own_requires_monomorphization(&self) -> bool {
+        for param in &self.params {
+            match param.kind {
+                GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
+                    return true;
+                }
+                GenericParamDefKind::Lifetime => {}
+            }
+        }
+        false
+    }
+
+    /// Returns the `GenericParamDef` with the given index.
+    pub fn param_at(&'tcx self, param_index: usize, tcx: TyCtxt<'tcx>) -> &'tcx GenericParamDef {
+        if let Some(index) = param_index.checked_sub(self.parent_count) {
+            &self.params[index]
+        } else {
+            tcx.generics_of(self.parent.expect("parent_count > 0 but no parent?"))
+                .param_at(param_index, tcx)
+        }
+    }
+
+    /// Returns the `GenericParamDef` associated with this `EarlyBoundRegion`.
+    pub fn region_param(
+        &'tcx self,
+        param: &EarlyBoundRegion,
+        tcx: TyCtxt<'tcx>,
+    ) -> &'tcx GenericParamDef {
+        let param = self.param_at(param.index as usize, tcx);
+        match param.kind {
+            GenericParamDefKind::Lifetime => param,
+            _ => bug!("expected lifetime parameter, but found another generic parameter"),
+        }
+    }
+
+    /// Returns the `GenericParamDef` associated with this `ParamTy`.
+    pub fn type_param(&'tcx self, param: &ParamTy, tcx: TyCtxt<'tcx>) -> &'tcx GenericParamDef {
+        let param = self.param_at(param.index as usize, tcx);
+        match param.kind {
+            GenericParamDefKind::Type { .. } => param,
+            _ => bug!("expected type parameter, but found another generic parameter"),
+        }
+    }
+
+    /// Returns the `GenericParamDef` associated with this `ParamConst`.
+    pub fn const_param(&'tcx self, param: &ParamConst, tcx: TyCtxt<'tcx>) -> &GenericParamDef {
+        let param = self.param_at(param.index as usize, tcx);
+        match param.kind {
+            GenericParamDefKind::Const { .. } => param,
+            _ => bug!("expected const parameter, but found another generic parameter"),
+        }
+    }
+}
+
+/// Bounds on generics.
+#[derive(Copy, Clone, Default, Debug, TyEncodable, TyDecodable, HashStable)]
+pub struct GenericPredicates<'tcx> {
+    pub parent: Option<DefId>,
+    pub predicates: &'tcx [(Predicate<'tcx>, Span)],
+}
+
+impl<'tcx> GenericPredicates<'tcx> {
+    pub fn instantiate(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        substs: SubstsRef<'tcx>,
+    ) -> InstantiatedPredicates<'tcx> {
+        let mut instantiated = InstantiatedPredicates::empty();
+        self.instantiate_into(tcx, &mut instantiated, substs);
+        instantiated
+    }
+
+    pub fn instantiate_own(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        substs: SubstsRef<'tcx>,
+    ) -> InstantiatedPredicates<'tcx> {
+        InstantiatedPredicates {
+            predicates: self.predicates.iter().map(|(p, _)| p.subst(tcx, substs)).collect(),
+            spans: self.predicates.iter().map(|(_, sp)| *sp).collect(),
+        }
+    }
+
+    fn instantiate_into(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        instantiated: &mut InstantiatedPredicates<'tcx>,
+        substs: SubstsRef<'tcx>,
+    ) {
+        if let Some(def_id) = self.parent {
+            tcx.predicates_of(def_id).instantiate_into(tcx, instantiated, substs);
+        }
+        instantiated.predicates.extend(self.predicates.iter().map(|(p, _)| p.subst(tcx, substs)));
+        instantiated.spans.extend(self.predicates.iter().map(|(_, sp)| *sp));
+    }
+
+    pub fn instantiate_identity(&self, tcx: TyCtxt<'tcx>) -> InstantiatedPredicates<'tcx> {
+        let mut instantiated = InstantiatedPredicates::empty();
+        self.instantiate_identity_into(tcx, &mut instantiated);
+        instantiated
+    }
+
+    fn instantiate_identity_into(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        instantiated: &mut InstantiatedPredicates<'tcx>,
+    ) {
+        if let Some(def_id) = self.parent {
+            tcx.predicates_of(def_id).instantiate_identity_into(tcx, instantiated);
+        }
+        instantiated.predicates.extend(self.predicates.iter().map(|(p, _)| p));
+        instantiated.spans.extend(self.predicates.iter().map(|(_, s)| s));
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index de012a69574..41d953216e0 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -216,9 +216,10 @@ impl<'tcx> InstanceDef<'tcx> {
             // drops of `Option::None` before LTO. We also respect the intent of
             // `#[inline]` on `Drop::drop` implementations.
             return ty.ty_adt_def().map_or(true, |adt_def| {
-                adt_def.destructor(tcx).map_or(adt_def.is_enum(), |dtor| {
-                    tcx.codegen_fn_attrs(dtor.did).requests_inline()
-                })
+                adt_def.destructor(tcx).map_or_else(
+                    || adt_def.is_enum(),
+                    |dtor| tcx.codegen_fn_attrs(dtor.did).requests_inline(),
+                )
             });
         }
         tcx.codegen_fn_attrs(self.def_id()).requests_inline()
@@ -482,6 +483,7 @@ impl<'tcx> Instance<'tcx> {
         if let Some(substs) = self.substs_for_mir_body() { v.subst(tcx, substs) } else { *v }
     }
 
+    #[inline(always)]
     pub fn subst_mir_and_normalize_erasing_regions<T>(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -499,7 +501,7 @@ impl<'tcx> Instance<'tcx> {
     }
 
     /// Returns a new `Instance` where generic parameters in `instance.substs` are replaced by
-    /// identify parameters if they are determined to be unused in `instance.def`.
+    /// identity parameters if they are determined to be unused in `instance.def`.
     pub fn polymorphize(self, tcx: TyCtxt<'tcx>) -> Self {
         debug!("polymorphize: running polymorphization analysis");
         if !tcx.sess.opts.debugging_opts.polymorphize {
@@ -593,7 +595,7 @@ fn polymorphize<'tcx>(
                 },
 
             // Simple case: If parameter is a const or type parameter..
-            ty::GenericParamDefKind::Const | ty::GenericParamDefKind::Type { .. } if
+            ty::GenericParamDefKind::Const { .. } | ty::GenericParamDefKind::Type { .. } if
                 // ..and is within range and unused..
                 unused.contains(param.index).unwrap_or(false) =>
                     // ..then use the identity for this parameter.
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 6d2ab0e5f5a..c2e9dba6c8e 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -1,3 +1,4 @@
+// ignore-tidy-filelength
 use crate::ich::StableHashingContext;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::mir::{GeneratorLayout, GeneratorSavedLocal};
@@ -11,7 +12,7 @@ use rustc_hir as hir;
 use rustc_hir::lang_items::LangItem;
 use rustc_index::bit_set::BitSet;
 use rustc_index::vec::{Idx, IndexVec};
-use rustc_session::{DataTypeKind, FieldInfo, SizeKind, VariantInfo};
+use rustc_session::{config::OptLevel, DataTypeKind, FieldInfo, SizeKind, VariantInfo};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::DUMMY_SP;
 use rustc_target::abi::call::{
@@ -1251,13 +1252,13 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 } else {
                     // Try to use a ScalarPair for all tagged enums.
                     let mut common_prim = None;
-                    for (field_layouts, layout_variant) in variants.iter().zip(&layout_variants) {
+                    for (field_layouts, layout_variant) in iter::zip(&variants, &layout_variants) {
                         let offsets = match layout_variant.fields {
                             FieldsShape::Arbitrary { ref offsets, .. } => offsets,
                             _ => bug!(),
                         };
                         let mut fields =
-                            field_layouts.iter().zip(offsets).filter(|p| !p.0.is_zst());
+                            iter::zip(field_layouts, offsets).filter(|p| !p.0.is_zst());
                         let (field, offset) = match (fields.next(), fields.next()) {
                             (None, None) => continue,
                             (Some(pair), None) => pair,
@@ -1626,7 +1627,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 const INVALID_FIELD_IDX: u32 = !0;
                 let mut combined_inverse_memory_index =
                     vec![INVALID_FIELD_IDX; promoted_memory_index.len() + memory_index.len()];
-                let mut offsets_and_memory_index = offsets.into_iter().zip(memory_index);
+                let mut offsets_and_memory_index = iter::zip(offsets, memory_index);
                 let combined_offsets = variant_fields
                     .iter()
                     .enumerate()
@@ -2318,31 +2319,30 @@ where
             ty::Ref(_, ty, mt) if offset.bytes() == 0 => {
                 let address_space = addr_space_of_ty(ty);
                 let tcx = cx.tcx();
-                let is_freeze = ty.is_freeze(tcx.at(DUMMY_SP), cx.param_env());
-                let kind = match mt {
-                    hir::Mutability::Not => {
-                        if is_freeze {
-                            PointerKind::Frozen
-                        } else {
-                            PointerKind::Shared
+                let kind = if tcx.sess.opts.optimize == OptLevel::No {
+                    // Use conservative pointer kind if not optimizing. This saves us the
+                    // Freeze/Unpin queries, and can save time in the codegen backend (noalias
+                    // attributes in LLVM have compile-time cost even in unoptimized builds).
+                    PointerKind::Shared
+                } else {
+                    match mt {
+                        hir::Mutability::Not => {
+                            if ty.is_freeze(tcx.at(DUMMY_SP), cx.param_env()) {
+                                PointerKind::Frozen
+                            } else {
+                                PointerKind::Shared
+                            }
                         }
-                    }
-                    hir::Mutability::Mut => {
-                        // Previously we would only emit noalias annotations for LLVM >= 6 or in
-                        // panic=abort mode. That was deemed right, as prior versions had many bugs
-                        // in conjunction with unwinding, but later versions didn’t seem to have
-                        // said issues. See issue #31681.
-                        //
-                        // Alas, later on we encountered a case where noalias would generate wrong
-                        // code altogether even with recent versions of LLVM in *safe* code with no
-                        // unwinding involved. See #54462.
-                        //
-                        // For now, do not enable mutable_noalias by default at all, while the
-                        // issue is being figured out.
-                        if tcx.sess.opts.debugging_opts.mutable_noalias {
-                            PointerKind::UniqueBorrowed
-                        } else {
-                            PointerKind::Shared
+                        hir::Mutability::Mut => {
+                            // References to self-referential structures should not be considered
+                            // noalias, as another pointer to the structure can be obtained, that
+                            // is not based-on the original reference. We consider all !Unpin
+                            // types to be potentially self-referential here.
+                            if ty.is_unpin(tcx.at(DUMMY_SP), cx.param_env()) {
+                                PointerKind::UniqueBorrowed
+                            } else {
+                                PointerKind::Shared
+                            }
                         }
                     }
                 };
@@ -2482,21 +2482,42 @@ impl<'tcx> ty::Instance<'tcx> {
             ty::Closure(def_id, substs) => {
                 let sig = substs.as_closure().sig();
 
-                let env_ty = tcx.closure_env_ty(def_id, substs).unwrap();
-                sig.map_bound(|sig| {
+                let bound_vars = tcx.mk_bound_variable_kinds(
+                    sig.bound_vars()
+                        .iter()
+                        .chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
+                );
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(bound_vars.len() - 1),
+                    kind: ty::BoundRegionKind::BrEnv,
+                };
+                let env_region = ty::ReLateBound(ty::INNERMOST, br);
+                let env_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap();
+
+                let sig = sig.skip_binder();
+                ty::Binder::bind_with_vars(
                     tcx.mk_fn_sig(
-                        iter::once(env_ty.skip_binder()).chain(sig.inputs().iter().cloned()),
+                        iter::once(env_ty).chain(sig.inputs().iter().cloned()),
                         sig.output(),
                         sig.c_variadic,
                         sig.unsafety,
                         sig.abi,
-                    )
-                })
+                    ),
+                    bound_vars,
+                )
             }
             ty::Generator(_, substs, _) => {
                 let sig = substs.as_generator().poly_sig();
 
-                let br = ty::BoundRegion { kind: ty::BrEnv };
+                let bound_vars = tcx.mk_bound_variable_kinds(
+                    sig.bound_vars()
+                        .iter()
+                        .chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
+                );
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(bound_vars.len() - 1),
+                    kind: ty::BoundRegionKind::BrEnv,
+                };
                 let env_region = ty::ReLateBound(ty::INNERMOST, br);
                 let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
 
@@ -2505,21 +2526,21 @@ impl<'tcx> ty::Instance<'tcx> {
                 let pin_substs = tcx.intern_substs(&[env_ty.into()]);
                 let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
 
-                sig.map_bound(|sig| {
-                    let state_did = tcx.require_lang_item(LangItem::GeneratorState, None);
-                    let state_adt_ref = tcx.adt_def(state_did);
-                    let state_substs =
-                        tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]);
-                    let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
-
+                let sig = sig.skip_binder();
+                let state_did = tcx.require_lang_item(LangItem::GeneratorState, None);
+                let state_adt_ref = tcx.adt_def(state_did);
+                let state_substs = tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]);
+                let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
+                ty::Binder::bind_with_vars(
                     tcx.mk_fn_sig(
                         [env_ty, sig.resume_ty].iter(),
                         &ret_ty,
                         false,
                         hir::Unsafety::Normal,
                         rustc_target::spec::abi::Abi::Rust,
-                    )
-                })
+                    ),
+                    bound_vars,
+                )
             }
             _ => bug!("unexpected type {:?} in Instance::fn_sig", ty),
         }
@@ -2562,6 +2583,7 @@ fn fn_can_unwind(
     panic_strategy: PanicStrategy,
     codegen_fn_attr_flags: CodegenFnAttrFlags,
     call_conv: Conv,
+    abi: SpecAbi,
 ) -> bool {
     if panic_strategy != PanicStrategy::Unwind {
         // In panic=abort mode we assume nothing can unwind anywhere, so
@@ -2586,17 +2608,35 @@ fn fn_can_unwind(
             //
             //  2. A Rust item using a non-Rust ABI (like `extern "C" fn foo() { ... }`).
             //
-            // Foreign items (case 1) are assumed to not unwind; it is
-            // UB otherwise. (At least for now; see also
-            // rust-lang/rust#63909 and Rust RFC 2753.)
-            //
-            // Items defined in Rust with non-Rust ABIs (case 2) are also
-            // not supposed to unwind. Whether this should be enforced
-            // (versus stating it is UB) and *how* it would be enforced
-            // is currently under discussion; see rust-lang/rust#58794.
-            //
-            // In either case, we mark item as explicitly nounwind.
-            false
+            // In both of these cases, we should refer to the ABI to determine whether or not we
+            // should unwind. See Rust RFC 2945 for more information on this behavior, here:
+            // https://github.com/rust-lang/rfcs/blob/master/text/2945-c-unwind-abi.md
+            use SpecAbi::*;
+            match abi {
+                C { unwind } | Stdcall { unwind } | System { unwind } | Thiscall { unwind } => {
+                    unwind
+                }
+                Cdecl
+                | Fastcall
+                | Vectorcall
+                | Aapcs
+                | Win64
+                | SysV64
+                | PtxKernel
+                | Msp430Interrupt
+                | X86Interrupt
+                | AmdGpuKernel
+                | EfiApi
+                | AvrInterrupt
+                | AvrNonBlockingInterrupt
+                | CCmseNonSecureCall
+                | Wasm
+                | RustIntrinsic
+                | PlatformIntrinsic
+                | Unadjusted => false,
+                // In the `if` above, we checked for functions with the Rust calling convention.
+                Rust | RustCall => unreachable!(),
+            }
         }
     }
 }
@@ -2654,14 +2694,14 @@ where
             RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust,
 
             // It's the ABI's job to select this, not ours.
-            System => bug!("system abi should be selected elsewhere"),
+            System { .. } => bug!("system abi should be selected elsewhere"),
             EfiApi => bug!("eficall abi should be selected elsewhere"),
 
-            Stdcall => Conv::X86Stdcall,
+            Stdcall { .. } => Conv::X86Stdcall,
             Fastcall => Conv::X86Fastcall,
             Vectorcall => Conv::X86VectorCall,
-            Thiscall => Conv::X86ThisCall,
-            C => Conv::C,
+            Thiscall { .. } => Conv::X86ThisCall,
+            C { .. } => Conv::C,
             Unadjusted => Conv::C,
             Win64 => Conv::X86_64Win64,
             SysV64 => Conv::X86_64SysV,
@@ -2673,6 +2713,7 @@ where
             AmdGpuKernel => Conv::AmdGpuKernel,
             AvrInterrupt => Conv::AvrInterrupt,
             AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt,
+            Wasm => Conv::C,
 
             // These API constants ought to be more specific...
             Cdecl => Conv::C,
@@ -2757,10 +2798,14 @@ where
                     // and can be marked as both `readonly` and `noalias`, as
                     // LLVM's definition of `noalias` is based solely on memory
                     // dependencies rather than pointer equality
+                    //
+                    // Due to miscompiles in LLVM < 12, we apply a separate NoAliasMutRef attribute
+                    // for UniqueBorrowed arguments, so that the codegen backend can decide
+                    // whether or not to actually emit the attribute.
                     let no_alias = match kind {
-                        PointerKind::Shared => false,
+                        PointerKind::Shared | PointerKind::UniqueBorrowed => false,
                         PointerKind::UniqueOwned => true,
-                        PointerKind::Frozen | PointerKind::UniqueBorrowed => !is_return,
+                        PointerKind::Frozen => !is_return,
                     };
                     if no_alias {
                         attrs.set(ArgAttribute::NoAlias);
@@ -2769,6 +2814,10 @@ where
                     if kind == PointerKind::Frozen && !is_return {
                         attrs.set(ArgAttribute::ReadOnly);
                     }
+
+                    if kind == PointerKind::UniqueBorrowed && !is_return {
+                        attrs.set(ArgAttribute::NoAliasMutRef);
+                    }
                 }
             }
         };
@@ -2823,7 +2872,12 @@ where
             c_variadic: sig.c_variadic,
             fixed_count: inputs.len(),
             conv,
-            can_unwind: fn_can_unwind(cx.tcx().sess.panic_strategy(), codegen_fn_attr_flags, conv),
+            can_unwind: fn_can_unwind(
+                cx.tcx().sess.panic_strategy(),
+                codegen_fn_attr_flags,
+                conv,
+                sig.abi,
+            ),
         };
         fn_abi.adjust_for_abi(cx, sig.abi);
         debug!("FnAbi::new_internal = {:?}", fn_abi);
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 2e077827873..6574c938260 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -9,105 +9,78 @@
 //!
 //! ["The `ty` module: representing types"]: https://rustc-dev-guide.rust-lang.org/ty.html
 
-// ignore-tidy-filelength
 pub use self::fold::{TypeFoldable, TypeFolder, TypeVisitor};
 pub use self::AssocItemContainer::*;
 pub use self::BorrowKind::*;
 pub use self::IntVarValue::*;
 pub use self::Variance::*;
+pub use adt::*;
+pub use assoc::*;
+pub use closure::*;
+pub use generics::*;
 
 use crate::hir::exports::ExportMap;
-use crate::hir::place::{
-    Place as HirPlace, PlaceBase as HirPlaceBase, ProjectionKind as HirProjectionKind,
-};
 use crate::ich::StableHashingContext;
 use crate::middle::cstore::CrateStoreDyn;
-use crate::middle::resolve_lifetime::ObjectLifetimeDefault;
-use crate::mir::interpret::ErrorHandled;
-use crate::mir::Body;
-use crate::mir::GeneratorLayout;
+use crate::mir::{Body, GeneratorLayout};
 use crate::traits::{self, Reveal};
 use crate::ty;
 use crate::ty::subst::{GenericArg, InternalSubsts, Subst, SubstsRef};
-use crate::ty::util::{Discr, IntTypeExt};
+use crate::ty::util::Discr;
 use rustc_ast as ast;
 use rustc_attr as attr;
 use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::fx::FxHashSet;
-use rustc_data_structures::fx::FxIndexMap;
-use rustc_data_structures::sorted_map::SortedIndexMultiMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{self, par_iter, ParallelIterator};
 use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
-use rustc_errors::ErrorReported;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, CtorOf, DefKind, Namespace, Res};
+use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, CRATE_DEF_INDEX};
-use rustc_hir::lang_items::LangItem;
 use rustc_hir::{Constness, Node};
-use rustc_index::vec::{Idx, IndexVec};
 use rustc_macros::HashStable;
-use rustc_serialize::{self, Encodable, Encoder};
-use rustc_session::DataTypeKind;
 use rustc_span::hygiene::ExpnId;
-use rustc_span::symbol::{kw, sym, Ident, Symbol};
+use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::Span;
-use rustc_target::abi::{Align, VariantIdx};
+use rustc_target::abi::Align;
 
-use std::cell::RefCell;
 use std::cmp::Ordering;
-use std::fmt;
 use std::hash::{Hash, Hasher};
-use std::ops::{ControlFlow, Range};
-use std::ptr;
-use std::str;
+use std::ops::ControlFlow;
+use std::{fmt, ptr, str};
 
-pub use self::sty::BoundRegionKind::*;
-pub use self::sty::RegionKind;
-pub use self::sty::RegionKind::*;
-pub use self::sty::TyKind::*;
-pub use self::sty::{Binder, BoundTy, BoundTyKind, BoundVar};
-pub use self::sty::{BoundRegion, BoundRegionKind, EarlyBoundRegion, FreeRegion, Region};
-pub use self::sty::{CanonicalPolyFnSig, FnSig, GenSig, PolyFnSig, PolyGenSig};
-pub use self::sty::{ClosureSubsts, GeneratorSubsts, TypeAndMut, UpvarSubsts};
-pub use self::sty::{ClosureSubstsParts, GeneratorSubstsParts};
-pub use self::sty::{ConstVid, RegionVid};
-pub use self::sty::{ExistentialPredicate, ParamConst, ParamTy, ProjectionTy};
-pub use self::sty::{ExistentialProjection, PolyExistentialProjection};
-pub use self::sty::{ExistentialTraitRef, PolyExistentialTraitRef};
-pub use self::sty::{PolyTraitRef, TraitRef, TyKind};
 pub use crate::ty::diagnostics::*;
 pub use rustc_type_ir::InferTy::*;
 pub use rustc_type_ir::*;
 
 pub use self::binding::BindingMode;
 pub use self::binding::BindingMode::*;
-
-pub use self::context::{tls, FreeRegionInfo, TyCtxt};
-pub use self::context::{
-    CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations,
-    DelaySpanBugEmitted, ResolvedOpaqueTy, UserType, UserTypeAnnotationIndex,
-};
+pub use self::consts::{Const, ConstInt, ConstKind, InferConst, ScalarInt, Unevaluated, ValTree};
 pub use self::context::{
-    CtxtInterners, GeneratorInteriorTypeCause, GlobalCtxt, Lift, TypeckResults,
+    tls, CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations,
+    CtxtInterners, DelaySpanBugEmitted, FreeRegionInfo, GeneratorInteriorTypeCause, GlobalCtxt,
+    Lift, ResolvedOpaqueTy, TyCtxt, TypeckResults, UserType, UserTypeAnnotationIndex,
 };
-
 pub use self::instance::{Instance, InstanceDef};
-
 pub use self::list::List;
-
+pub use self::sty::BoundRegionKind::*;
+pub use self::sty::RegionKind::*;
+pub use self::sty::TyKind::*;
+pub use self::sty::{
+    Binder, BoundRegion, BoundRegionKind, BoundTy, BoundTyKind, BoundVar, BoundVariableKind,
+    CanonicalPolyFnSig, ClosureSubsts, ClosureSubstsParts, ConstVid, EarlyBoundRegion,
+    ExistentialPredicate, ExistentialProjection, ExistentialTraitRef, FnSig, FreeRegion, GenSig,
+    GeneratorSubsts, GeneratorSubstsParts, ParamConst, ParamTy, PolyExistentialProjection,
+    PolyExistentialTraitRef, PolyFnSig, PolyGenSig, PolyTraitRef, ProjectionTy, Region, RegionKind,
+    RegionVid, TraitRef, TyKind, TypeAndMut, UpvarSubsts,
+};
 pub use self::trait_def::TraitDef;
 
-pub use self::consts::{Const, ConstInt, ConstKind, InferConst, ScalarInt};
-
 pub mod _match;
 pub mod adjustment;
 pub mod binding;
 pub mod cast;
 pub mod codec;
-mod erase_regions;
 pub mod error;
 pub mod fast_reject;
 pub mod flags;
@@ -124,9 +97,14 @@ pub mod trait_def;
 pub mod util;
 pub mod walk;
 
+mod adt;
+mod assoc;
+mod closure;
 mod consts;
 mod context;
 mod diagnostics;
+mod erase_regions;
+mod generics;
 mod instance;
 mod list;
 mod structural_impls;
@@ -148,30 +126,6 @@ pub struct ResolverOutputs {
     pub extern_prelude: FxHashMap<Symbol, bool>,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug, HashStable, Hash)]
-pub enum AssocItemContainer {
-    TraitContainer(DefId),
-    ImplContainer(DefId),
-}
-
-impl AssocItemContainer {
-    /// Asserts that this is the `DefId` of an associated item declared
-    /// in a trait, and returns the trait `DefId`.
-    pub fn assert_trait(&self) -> DefId {
-        match *self {
-            TraitContainer(id) => id,
-            _ => bug!("associated item has wrong container type: {:?}", self),
-        }
-    }
-
-    pub fn id(&self) -> DefId {
-        match *self {
-            TraitContainer(id) => id,
-            ImplContainer(id) => id,
-        }
-    }
-}
-
 /// The "header" of an impl is everything outside the body: a Self type, a trait
 /// ref (in the case of a trait impl), and a set of predicates (from the
 /// bounds / where-clauses).
@@ -196,142 +150,6 @@ pub enum ImplPolarity {
     Reservation,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, HashStable, Eq, Hash)]
-pub struct AssocItem {
-    pub def_id: DefId,
-    #[stable_hasher(project(name))]
-    pub ident: Ident,
-    pub kind: AssocKind,
-    pub vis: Visibility,
-    pub defaultness: hir::Defaultness,
-    pub container: AssocItemContainer,
-
-    /// Whether this is a method with an explicit self
-    /// as its first parameter, allowing method calls.
-    pub fn_has_self_parameter: bool,
-}
-
-#[derive(Copy, Clone, PartialEq, Debug, HashStable, Eq, Hash)]
-pub enum AssocKind {
-    Const,
-    Fn,
-    Type,
-}
-
-impl AssocKind {
-    pub fn namespace(&self) -> Namespace {
-        match *self {
-            ty::AssocKind::Type => Namespace::TypeNS,
-            ty::AssocKind::Const | ty::AssocKind::Fn => Namespace::ValueNS,
-        }
-    }
-
-    pub fn as_def_kind(&self) -> DefKind {
-        match self {
-            AssocKind::Const => DefKind::AssocConst,
-            AssocKind::Fn => DefKind::AssocFn,
-            AssocKind::Type => DefKind::AssocTy,
-        }
-    }
-}
-
-impl AssocItem {
-    pub fn signature(&self, tcx: TyCtxt<'_>) -> String {
-        match self.kind {
-            ty::AssocKind::Fn => {
-                // We skip the binder here because the binder would deanonymize all
-                // late-bound regions, and we don't want method signatures to show up
-                // `as for<'r> fn(&'r MyType)`.  Pretty-printing handles late-bound
-                // regions just fine, showing `fn(&MyType)`.
-                tcx.fn_sig(self.def_id).skip_binder().to_string()
-            }
-            ty::AssocKind::Type => format!("type {};", self.ident),
-            ty::AssocKind::Const => {
-                format!("const {}: {:?};", self.ident, tcx.type_of(self.def_id))
-            }
-        }
-    }
-}
-
-/// A list of `ty::AssocItem`s in definition order that allows for efficient lookup by name.
-///
-/// When doing lookup by name, we try to postpone hygienic comparison for as long as possible since
-/// it is relatively expensive. Instead, items are indexed by `Symbol` and hygienic comparison is
-/// done only on items with the same name.
-#[derive(Debug, Clone, PartialEq, HashStable)]
-pub struct AssociatedItems<'tcx> {
-    items: SortedIndexMultiMap<u32, Symbol, &'tcx ty::AssocItem>,
-}
-
-impl<'tcx> AssociatedItems<'tcx> {
-    /// Constructs an `AssociatedItems` map from a series of `ty::AssocItem`s in definition order.
-    pub fn new(items_in_def_order: impl IntoIterator<Item = &'tcx ty::AssocItem>) -> Self {
-        let items = items_in_def_order.into_iter().map(|item| (item.ident.name, item)).collect();
-        AssociatedItems { items }
-    }
-
-    /// Returns a slice of associated items in the order they were defined.
-    ///
-    /// New code should avoid relying on definition order. If you need a particular associated item
-    /// for a known trait, make that trait a lang item instead of indexing this array.
-    pub fn in_definition_order(&self) -> impl '_ + Iterator<Item = &ty::AssocItem> {
-        self.items.iter().map(|(_, v)| *v)
-    }
-
-    pub fn len(&self) -> usize {
-        self.items.len()
-    }
-
-    /// Returns an iterator over all associated items with the given name, ignoring hygiene.
-    pub fn filter_by_name_unhygienic(
-        &self,
-        name: Symbol,
-    ) -> impl '_ + Iterator<Item = &ty::AssocItem> {
-        self.items.get_by_key(&name).copied()
-    }
-
-    /// Returns an iterator over all associated items with the given name.
-    ///
-    /// Multiple items may have the same name if they are in different `Namespace`s. For example,
-    /// an associated type can have the same name as a method. Use one of the `find_by_name_and_*`
-    /// methods below if you know which item you are looking for.
-    pub fn filter_by_name(
-        &'a self,
-        tcx: TyCtxt<'a>,
-        ident: Ident,
-        parent_def_id: DefId,
-    ) -> impl 'a + Iterator<Item = &'a ty::AssocItem> {
-        self.filter_by_name_unhygienic(ident.name)
-            .filter(move |item| tcx.hygienic_eq(ident, item.ident, parent_def_id))
-    }
-
-    /// Returns the associated item with the given name and `AssocKind`, if one exists.
-    pub fn find_by_name_and_kind(
-        &self,
-        tcx: TyCtxt<'_>,
-        ident: Ident,
-        kind: AssocKind,
-        parent_def_id: DefId,
-    ) -> Option<&ty::AssocItem> {
-        self.filter_by_name_unhygienic(ident.name)
-            .filter(|item| item.kind == kind)
-            .find(|item| tcx.hygienic_eq(ident, item.ident, parent_def_id))
-    }
-
-    /// Returns the associated item with the given name in the given `Namespace`, if one exists.
-    pub fn find_by_name_and_namespace(
-        &self,
-        tcx: TyCtxt<'_>,
-        ident: Ident,
-        ns: Namespace,
-        parent_def_id: DefId,
-    ) -> Option<&ty::AssocItem> {
-        self.filter_by_name_unhygienic(ident.name)
-            .filter(|item| item.kind.namespace() == ns)
-            .find(|item| tcx.hygienic_eq(ident, item.ident, parent_def_id))
-    }
-}
-
 #[derive(Clone, Debug, PartialEq, Eq, Copy, Hash, TyEncodable, TyDecodable, HashStable)]
 pub enum Visibility {
     /// Visible everywhere (including in other crates).
@@ -483,8 +301,8 @@ impl<'tcx> TyS<'tcx> {
 }
 
 // `TyS` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
-static_assert_size!(TyS<'_>, 32);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+static_assert_size!(TyS<'_>, 40);
 
 impl<'tcx> Ord for TyS<'tcx> {
     fn cmp(&self, other: &TyS<'tcx>) -> Ordering {
@@ -531,243 +349,6 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for TyS<'tcx> {
 #[rustc_diagnostic_item = "Ty"]
 pub type Ty<'tcx> = &'tcx TyS<'tcx>;
 
-#[derive(
-    Clone,
-    Copy,
-    Debug,
-    PartialEq,
-    Eq,
-    Hash,
-    TyEncodable,
-    TyDecodable,
-    TypeFoldable,
-    HashStable
-)]
-pub struct UpvarPath {
-    pub hir_id: hir::HirId,
-}
-
-/// Upvars do not get their own `NodeId`. Instead, we use the pair of
-/// the original var ID (that is, the root variable that is referenced
-/// by the upvar) and the ID of the closure expression.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
-pub struct UpvarId {
-    pub var_path: UpvarPath,
-    pub closure_expr_id: LocalDefId,
-}
-
-impl UpvarId {
-    pub fn new(var_hir_id: hir::HirId, closure_def_id: LocalDefId) -> UpvarId {
-        UpvarId { var_path: UpvarPath { hir_id: var_hir_id }, closure_expr_id: closure_def_id }
-    }
-}
-
-#[derive(Clone, PartialEq, Debug, TyEncodable, TyDecodable, TypeFoldable, Copy, HashStable)]
-pub enum BorrowKind {
-    /// Data must be immutable and is aliasable.
-    ImmBorrow,
-
-    /// Data must be immutable but not aliasable. This kind of borrow
-    /// cannot currently be expressed by the user and is used only in
-    /// implicit closure bindings. It is needed when the closure
-    /// is borrowing or mutating a mutable referent, e.g.:
-    ///
-    /// ```
-    /// let x: &mut isize = ...;
-    /// let y = || *x += 5;
-    /// ```
-    ///
-    /// If we were to try to translate this closure into a more explicit
-    /// form, we'd encounter an error with the code as written:
-    ///
-    /// ```
-    /// struct Env { x: & &mut isize }
-    /// let x: &mut isize = ...;
-    /// let y = (&mut Env { &x }, fn_ptr);  // Closure is pair of env and fn
-    /// fn fn_ptr(env: &mut Env) { **env.x += 5; }
-    /// ```
-    ///
-    /// This is then illegal because you cannot mutate a `&mut` found
-    /// in an aliasable location. To solve, you'd have to translate with
-    /// an `&mut` borrow:
-    ///
-    /// ```
-    /// struct Env { x: & &mut isize }
-    /// let x: &mut isize = ...;
-    /// let y = (&mut Env { &mut x }, fn_ptr); // changed from &x to &mut x
-    /// fn fn_ptr(env: &mut Env) { **env.x += 5; }
-    /// ```
-    ///
-    /// Now the assignment to `**env.x` is legal, but creating a
-    /// mutable pointer to `x` is not because `x` is not mutable. We
-    /// could fix this by declaring `x` as `let mut x`. This is ok in
-    /// user code, if awkward, but extra weird for closures, since the
-    /// borrow is hidden.
-    ///
-    /// So we introduce a "unique imm" borrow -- the referent is
-    /// immutable, but not aliasable. This solves the problem. For
-    /// simplicity, we don't give users the way to express this
-    /// borrow, it's just used when translating closures.
-    UniqueImmBorrow,
-
-    /// Data is mutable and not aliasable.
-    MutBorrow,
-}
-
-/// Information describing the capture of an upvar. This is computed
-/// during `typeck`, specifically by `regionck`.
-#[derive(PartialEq, Clone, Debug, Copy, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
-pub enum UpvarCapture<'tcx> {
-    /// Upvar is captured by value. This is always true when the
-    /// closure is labeled `move`, but can also be true in other cases
-    /// depending on inference.
-    ///
-    /// If the upvar was inferred to be captured by value (e.g. `move`
-    /// was not used), then the `Span` points to a usage that
-    /// required it. There may be more than one such usage
-    /// (e.g. `|| { a; a; }`), in which case we pick an
-    /// arbitrary one.
-    ByValue(Option<Span>),
-
-    /// Upvar is captured by reference.
-    ByRef(UpvarBorrow<'tcx>),
-}
-
-#[derive(PartialEq, Clone, Copy, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
-pub struct UpvarBorrow<'tcx> {
-    /// The kind of borrow: by-ref upvars have access to shared
-    /// immutable borrows, which are not part of the normal language
-    /// syntax.
-    pub kind: BorrowKind,
-
-    /// Region of the resulting reference.
-    pub region: ty::Region<'tcx>,
-}
-
-/// Given the closure DefId this map provides a map of root variables to minimum
-/// set of `CapturedPlace`s that need to be tracked to support all captures of that closure.
-pub type MinCaptureInformationMap<'tcx> = FxHashMap<DefId, RootVariableMinCaptureList<'tcx>>;
-
-/// Part of `MinCaptureInformationMap`; Maps a root variable to the list of `CapturedPlace`.
-/// Used to track the minimum set of `Place`s that need to be captured to support all
-/// Places captured by the closure starting at a given root variable.
-///
-/// This provides a convenient and quick way of checking if a variable being used within
-/// a closure is a capture of a local variable.
-pub type RootVariableMinCaptureList<'tcx> = FxIndexMap<hir::HirId, MinCaptureList<'tcx>>;
-
-/// Part of `MinCaptureInformationMap`; List of `CapturePlace`s.
-pub type MinCaptureList<'tcx> = Vec<CapturedPlace<'tcx>>;
-
-/// A composite describing a `Place` that is captured by a closure.
-#[derive(PartialEq, Clone, Debug, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
-pub struct CapturedPlace<'tcx> {
-    /// The `Place` that is captured.
-    pub place: HirPlace<'tcx>,
-
-    /// `CaptureKind` and expression(s) that resulted in such capture of `place`.
-    pub info: CaptureInfo<'tcx>,
-
-    /// Represents if `place` can be mutated or not.
-    pub mutability: hir::Mutability,
-}
-
-impl CapturedPlace<'tcx> {
-    /// Returns the hir-id of the root variable for the captured place.
-    /// e.g., if `a.b.c` was captured, would return the hir-id for `a`.
-    pub fn get_root_variable(&self) -> hir::HirId {
-        match self.place.base {
-            HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
-            base => bug!("Expected upvar, found={:?}", base),
-        }
-    }
-}
-
-pub fn place_to_string_for_capture(tcx: TyCtxt<'tcx>, place: &HirPlace<'tcx>) -> String {
-    let name = match place.base {
-        HirPlaceBase::Upvar(upvar_id) => tcx.hir().name(upvar_id.var_path.hir_id).to_string(),
-        _ => bug!("Capture_information should only contain upvars"),
-    };
-    let mut curr_string = name;
-
-    for (i, proj) in place.projections.iter().enumerate() {
-        match proj.kind {
-            HirProjectionKind::Deref => {
-                curr_string = format!("*{}", curr_string);
-            }
-            HirProjectionKind::Field(idx, variant) => match place.ty_before_projection(i).kind() {
-                ty::Adt(def, ..) => {
-                    curr_string = format!(
-                        "{}.{}",
-                        curr_string,
-                        def.variants[variant].fields[idx as usize].ident.name.as_str()
-                    );
-                }
-                ty::Tuple(_) => {
-                    curr_string = format!("{}.{}", curr_string, idx);
-                }
-                _ => {
-                    bug!(
-                        "Field projection applied to a type other than Adt or Tuple: {:?}.",
-                        place.ty_before_projection(i).kind()
-                    )
-                }
-            },
-            proj => bug!("{:?} unexpected because it isn't captured", proj),
-        }
-    }
-
-    curr_string.to_string()
-}
-
-/// Part of `MinCaptureInformationMap`; describes the capture kind (&, &mut, move)
-/// for a particular capture as well as identifying the part of the source code
-/// that triggered this capture to occur.
-#[derive(PartialEq, Clone, Debug, Copy, TyEncodable, TyDecodable, TypeFoldable, HashStable)]
-pub struct CaptureInfo<'tcx> {
-    /// Expr Id pointing to use that resulted in selecting the current capture kind
-    ///
-    /// Eg:
-    /// ```rust,no_run
-    /// let mut t = (0,1);
-    ///
-    /// let c = || {
-    ///     println!("{}",t); // L1
-    ///     t.1 = 4; // L2
-    /// };
-    /// ```
-    /// `capture_kind_expr_id` will point to the use on L2 and `path_expr_id` will point to the
-    /// use on L1.
-    ///
-    /// If the user doesn't enable feature `capture_disjoint_fields` (RFC 2229) then, it is
-    /// possible that we don't see the use of a particular place resulting in capture_kind_expr_id being
-    /// None. In such case we fallback on uvpars_mentioned for span.
-    ///
-    /// Eg:
-    /// ```rust,no_run
-    /// let x = 5;
-    ///
-    /// let c = || {
-    ///     let _ = x
-    /// };
-    /// ```
-    ///
-    /// In this example, if `capture_disjoint_fields` is **not** set, then x will be captured,
-    /// but we won't see it being used during capture analysis, since it's essentially a discard.
-    pub capture_kind_expr_id: Option<hir::HirId>,
-    /// Expr Id pointing to use that resulted the corresponding place being captured
-    ///
-    /// See `capture_kind_expr_id` for example.
-    ///
-    pub path_expr_id: Option<hir::HirId>,
-
-    /// Capture mode that was selected
-    pub capture_kind: UpvarCapture<'tcx>,
-}
-
-pub type UpvarListMap = FxHashMap<DefId, FxIndexMap<hir::HirId, UpvarId>>;
-pub type UpvarCaptureMap<'tcx> = FxHashMap<UpvarId, UpvarCapture<'tcx>>;
-
 impl ty::EarlyBoundRegion {
     /// Does this early bound region have a name? Early bound regions normally
     /// always have names except when using anonymous lifetimes (`'_`).
@@ -776,262 +357,16 @@ impl ty::EarlyBoundRegion {
     }
 }
 
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub enum GenericParamDefKind {
-    Lifetime,
-    Type {
-        has_default: bool,
-        object_lifetime_default: ObjectLifetimeDefault,
-        synthetic: Option<hir::SyntheticTyParamKind>,
-    },
-    Const,
-}
-
-impl GenericParamDefKind {
-    pub fn descr(&self) -> &'static str {
-        match self {
-            GenericParamDefKind::Lifetime => "lifetime",
-            GenericParamDefKind::Type { .. } => "type",
-            GenericParamDefKind::Const => "constant",
-        }
-    }
-    pub fn to_ord(&self, tcx: TyCtxt<'_>) -> ast::ParamKindOrd {
-        match self {
-            GenericParamDefKind::Lifetime => ast::ParamKindOrd::Lifetime,
-            GenericParamDefKind::Type { .. } => ast::ParamKindOrd::Type,
-            GenericParamDefKind::Const => {
-                ast::ParamKindOrd::Const { unordered: tcx.features().const_generics }
-            }
-        }
-    }
-}
-
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct GenericParamDef {
-    pub name: Symbol,
-    pub def_id: DefId,
-    pub index: u32,
-
-    /// `pure_wrt_drop`, set by the (unsafe) `#[may_dangle]` attribute
-    /// on generic parameter `'a`/`T`, asserts data behind the parameter
-    /// `'a`/`T` won't be accessed during the parent type's `Drop` impl.
-    pub pure_wrt_drop: bool,
-
-    pub kind: GenericParamDefKind,
-}
-
-impl GenericParamDef {
-    pub fn to_early_bound_region_data(&self) -> ty::EarlyBoundRegion {
-        if let GenericParamDefKind::Lifetime = self.kind {
-            ty::EarlyBoundRegion { def_id: self.def_id, index: self.index, name: self.name }
-        } else {
-            bug!("cannot convert a non-lifetime parameter def to an early bound region")
-        }
-    }
-}
-
-#[derive(Default)]
-pub struct GenericParamCount {
-    pub lifetimes: usize,
-    pub types: usize,
-    pub consts: usize,
-}
-
-/// Information about the formal type/lifetime parameters associated
-/// with an item or method. Analogous to `hir::Generics`.
-///
-/// The ordering of parameters is the same as in `Subst` (excluding child generics):
-/// `Self` (optionally), `Lifetime` params..., `Type` params...
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct Generics {
-    pub parent: Option<DefId>,
-    pub parent_count: usize,
-    pub params: Vec<GenericParamDef>,
-
-    /// Reverse map to the `index` field of each `GenericParamDef`.
-    #[stable_hasher(ignore)]
-    pub param_def_id_to_index: FxHashMap<DefId, u32>,
-
-    pub has_self: bool,
-    pub has_late_bound_regions: Option<Span>,
-}
-
-impl<'tcx> Generics {
-    pub fn count(&self) -> usize {
-        self.parent_count + self.params.len()
-    }
-
-    pub fn own_counts(&self) -> GenericParamCount {
-        // We could cache this as a property of `GenericParamCount`, but
-        // the aim is to refactor this away entirely eventually and the
-        // presence of this method will be a constant reminder.
-        let mut own_counts = GenericParamCount::default();
-
-        for param in &self.params {
-            match param.kind {
-                GenericParamDefKind::Lifetime => own_counts.lifetimes += 1,
-                GenericParamDefKind::Type { .. } => own_counts.types += 1,
-                GenericParamDefKind::Const => own_counts.consts += 1,
-            }
-        }
-
-        own_counts
-    }
-
-    pub fn own_defaults(&self) -> GenericParamCount {
-        let mut own_defaults = GenericParamCount::default();
-
-        for param in &self.params {
-            match param.kind {
-                GenericParamDefKind::Lifetime => (),
-                GenericParamDefKind::Type { has_default, .. } => {
-                    own_defaults.types += has_default as usize;
-                }
-                GenericParamDefKind::Const => {
-                    // FIXME(const_generics:defaults)
-                }
-            }
-        }
-
-        own_defaults
-    }
-
-    pub fn requires_monomorphization(&self, tcx: TyCtxt<'tcx>) -> bool {
-        if self.own_requires_monomorphization() {
-            return true;
-        }
-
-        if let Some(parent_def_id) = self.parent {
-            let parent = tcx.generics_of(parent_def_id);
-            parent.requires_monomorphization(tcx)
-        } else {
-            false
-        }
-    }
-
-    pub fn own_requires_monomorphization(&self) -> bool {
-        for param in &self.params {
-            match param.kind {
-                GenericParamDefKind::Type { .. } | GenericParamDefKind::Const => return true,
-                GenericParamDefKind::Lifetime => {}
-            }
-        }
-        false
-    }
-
-    /// Returns the `GenericParamDef` with the given index.
-    pub fn param_at(&'tcx self, param_index: usize, tcx: TyCtxt<'tcx>) -> &'tcx GenericParamDef {
-        if let Some(index) = param_index.checked_sub(self.parent_count) {
-            &self.params[index]
-        } else {
-            tcx.generics_of(self.parent.expect("parent_count > 0 but no parent?"))
-                .param_at(param_index, tcx)
-        }
-    }
-
-    /// Returns the `GenericParamDef` associated with this `EarlyBoundRegion`.
-    pub fn region_param(
-        &'tcx self,
-        param: &EarlyBoundRegion,
-        tcx: TyCtxt<'tcx>,
-    ) -> &'tcx GenericParamDef {
-        let param = self.param_at(param.index as usize, tcx);
-        match param.kind {
-            GenericParamDefKind::Lifetime => param,
-            _ => bug!("expected lifetime parameter, but found another generic parameter"),
-        }
-    }
-
-    /// Returns the `GenericParamDef` associated with this `ParamTy`.
-    pub fn type_param(&'tcx self, param: &ParamTy, tcx: TyCtxt<'tcx>) -> &'tcx GenericParamDef {
-        let param = self.param_at(param.index as usize, tcx);
-        match param.kind {
-            GenericParamDefKind::Type { .. } => param,
-            _ => bug!("expected type parameter, but found another generic parameter"),
-        }
-    }
-
-    /// Returns the `GenericParamDef` associated with this `ParamConst`.
-    pub fn const_param(&'tcx self, param: &ParamConst, tcx: TyCtxt<'tcx>) -> &GenericParamDef {
-        let param = self.param_at(param.index as usize, tcx);
-        match param.kind {
-            GenericParamDefKind::Const => param,
-            _ => bug!("expected const parameter, but found another generic parameter"),
-        }
-    }
-}
-
-/// Bounds on generics.
-#[derive(Copy, Clone, Default, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct GenericPredicates<'tcx> {
-    pub parent: Option<DefId>,
-    pub predicates: &'tcx [(Predicate<'tcx>, Span)],
-}
-
-impl<'tcx> GenericPredicates<'tcx> {
-    pub fn instantiate(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        substs: SubstsRef<'tcx>,
-    ) -> InstantiatedPredicates<'tcx> {
-        let mut instantiated = InstantiatedPredicates::empty();
-        self.instantiate_into(tcx, &mut instantiated, substs);
-        instantiated
-    }
-
-    pub fn instantiate_own(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        substs: SubstsRef<'tcx>,
-    ) -> InstantiatedPredicates<'tcx> {
-        InstantiatedPredicates {
-            predicates: self.predicates.iter().map(|(p, _)| p.subst(tcx, substs)).collect(),
-            spans: self.predicates.iter().map(|(_, sp)| *sp).collect(),
-        }
-    }
-
-    fn instantiate_into(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        instantiated: &mut InstantiatedPredicates<'tcx>,
-        substs: SubstsRef<'tcx>,
-    ) {
-        if let Some(def_id) = self.parent {
-            tcx.predicates_of(def_id).instantiate_into(tcx, instantiated, substs);
-        }
-        instantiated.predicates.extend(self.predicates.iter().map(|(p, _)| p.subst(tcx, substs)));
-        instantiated.spans.extend(self.predicates.iter().map(|(_, sp)| *sp));
-    }
-
-    pub fn instantiate_identity(&self, tcx: TyCtxt<'tcx>) -> InstantiatedPredicates<'tcx> {
-        let mut instantiated = InstantiatedPredicates::empty();
-        self.instantiate_identity_into(tcx, &mut instantiated);
-        instantiated
-    }
-
-    fn instantiate_identity_into(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        instantiated: &mut InstantiatedPredicates<'tcx>,
-    ) {
-        if let Some(def_id) = self.parent {
-            tcx.predicates_of(def_id).instantiate_identity_into(tcx, instantiated);
-        }
-        instantiated.predicates.extend(self.predicates.iter().map(|(p, _)| p));
-        instantiated.spans.extend(self.predicates.iter().map(|(_, s)| s));
-    }
-}
-
 #[derive(Debug)]
 crate struct PredicateInner<'tcx> {
-    kind: Binder<PredicateKind<'tcx>>,
+    kind: Binder<'tcx, PredicateKind<'tcx>>,
     flags: TypeFlags,
     /// See the comment for the corresponding field of [TyS].
     outer_exclusive_binder: ty::DebruijnIndex,
 }
 
-#[cfg(target_arch = "x86_64")]
-static_assert_size!(PredicateInner<'_>, 40);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+static_assert_size!(PredicateInner<'_>, 48);
 
 #[derive(Clone, Copy, Lift)]
 pub struct Predicate<'tcx> {
@@ -1054,9 +389,9 @@ impl Hash for Predicate<'_> {
 impl<'tcx> Eq for Predicate<'tcx> {}
 
 impl<'tcx> Predicate<'tcx> {
-    /// Gets the inner `Binder<PredicateKind<'tcx>>`.
+    /// Gets the inner `Binder<'tcx, PredicateKind<'tcx>>`.
     #[inline]
-    pub fn kind(self) -> Binder<PredicateKind<'tcx>> {
+    pub fn kind(self) -> Binder<'tcx, PredicateKind<'tcx>> {
         self.inner.kind
     }
 }
@@ -1208,10 +543,33 @@ impl<'tcx> Predicate<'tcx> {
         // substitution code expects equal binding levels in the values
         // from the substitution and the value being substituted into, and
         // this trick achieves that).
-        let substs = trait_ref.skip_binder().substs;
-        let pred = self.kind().skip_binder();
-        let new = pred.subst(tcx, substs);
-        tcx.reuse_or_mk_predicate(self, ty::Binder::bind(new))
+
+        // Working through the second example:
+        // trait_ref: for<'x> T: Foo1<'^0.0>; substs: [T, '^0.0]
+        // predicate: for<'b> Self: Bar1<'a, '^0.0>; substs: [Self, 'a, '^0.0]
+        // We want to end up with:
+        //     for<'x, 'b> T: Bar1<'^0.0, '^0.1>
+        // To do this:
+        // 1) We must shift all bound vars in predicate by the length
+        //    of trait ref's bound vars. So, we would end up with predicate like
+        //    Self: Bar1<'a, '^0.1>
+        // 2) We can then apply the trait substs to this, ending up with
+        //    T: Bar1<'^0.0, '^0.1>
+        // 3) Finally, to create the final bound vars, we concatenate the bound
+        //    vars of the trait ref with those of the predicate:
+        //    ['x, 'b]
+        let bound_pred = self.kind();
+        let pred_bound_vars = bound_pred.bound_vars();
+        let trait_bound_vars = trait_ref.bound_vars();
+        // 1) Self: Bar1<'a, '^0.0> -> Self: Bar1<'a, '^0.1>
+        let shifted_pred =
+            tcx.shift_bound_var_indices(trait_bound_vars.len(), bound_pred.skip_binder());
+        // 2) Self: Bar1<'a, '^0.1> -> T: Bar1<'^0.0, '^0.1>
+        let new = shifted_pred.subst(tcx, trait_ref.skip_binder().substs);
+        // 3) ['x] + ['b] -> ['x, 'b]
+        let bound_vars =
+            tcx.mk_bound_variable_kinds(trait_bound_vars.iter().chain(pred_bound_vars));
+        tcx.reuse_or_mk_predicate(self, ty::Binder::bind_with_vars(new, bound_vars))
     }
 }
 
@@ -1221,7 +579,7 @@ pub struct TraitPredicate<'tcx> {
     pub trait_ref: TraitRef<'tcx>,
 }
 
-pub type PolyTraitPredicate<'tcx> = ty::Binder<TraitPredicate<'tcx>>;
+pub type PolyTraitPredicate<'tcx> = ty::Binder<'tcx, TraitPredicate<'tcx>>;
 
 impl<'tcx> TraitPredicate<'tcx> {
     pub fn def_id(self) -> DefId {
@@ -1239,7 +597,7 @@ impl<'tcx> PolyTraitPredicate<'tcx> {
         self.skip_binder().def_id()
     }
 
-    pub fn self_ty(self) -> ty::Binder<Ty<'tcx>> {
+    pub fn self_ty(self) -> ty::Binder<'tcx, Ty<'tcx>> {
         self.map_bound(|trait_ref| trait_ref.self_ty())
     }
 }
@@ -1249,8 +607,8 @@ impl<'tcx> PolyTraitPredicate<'tcx> {
 pub struct OutlivesPredicate<A, B>(pub A, pub B); // `A: B`
 pub type RegionOutlivesPredicate<'tcx> = OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>;
 pub type TypeOutlivesPredicate<'tcx> = OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>;
-pub type PolyRegionOutlivesPredicate<'tcx> = ty::Binder<RegionOutlivesPredicate<'tcx>>;
-pub type PolyTypeOutlivesPredicate<'tcx> = ty::Binder<TypeOutlivesPredicate<'tcx>>;
+pub type PolyRegionOutlivesPredicate<'tcx> = ty::Binder<'tcx, RegionOutlivesPredicate<'tcx>>;
+pub type PolyTypeOutlivesPredicate<'tcx> = ty::Binder<'tcx, TypeOutlivesPredicate<'tcx>>;
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
 #[derive(HashStable, TypeFoldable)]
@@ -1259,7 +617,7 @@ pub struct SubtypePredicate<'tcx> {
     pub a: Ty<'tcx>,
     pub b: Ty<'tcx>,
 }
-pub type PolySubtypePredicate<'tcx> = ty::Binder<SubtypePredicate<'tcx>>;
+pub type PolySubtypePredicate<'tcx> = ty::Binder<'tcx, SubtypePredicate<'tcx>>;
 
 /// This kind of predicate has no *direct* correspondent in the
 /// syntax, but it roughly corresponds to the syntactic forms:
@@ -1280,25 +638,15 @@ pub struct ProjectionPredicate<'tcx> {
     pub ty: Ty<'tcx>,
 }
 
-pub type PolyProjectionPredicate<'tcx> = Binder<ProjectionPredicate<'tcx>>;
+pub type PolyProjectionPredicate<'tcx> = Binder<'tcx, ProjectionPredicate<'tcx>>;
 
 impl<'tcx> PolyProjectionPredicate<'tcx> {
-    /// Returns the `DefId` of the associated item being projected.
-    pub fn item_def_id(&self) -> DefId {
-        self.skip_binder().projection_ty.item_def_id
-    }
-
     /// Returns the `DefId` of the trait of the associated item being projected.
     #[inline]
     pub fn trait_def_id(&self, tcx: TyCtxt<'tcx>) -> DefId {
         self.skip_binder().projection_ty.trait_def_id(tcx)
     }
 
-    #[inline]
-    pub fn projection_self_ty(&self) -> Binder<Ty<'tcx>> {
-        self.map_bound(|predicate| predicate.projection_ty.self_ty())
-    }
-
     /// Get the [PolyTraitRef] required for this projection to be well formed.
     /// Note that for generic associated types the predicates of the associated
     /// type also need to be checked.
@@ -1312,7 +660,7 @@ impl<'tcx> PolyProjectionPredicate<'tcx> {
         self.map_bound(|predicate| predicate.projection_ty.trait_ref(tcx))
     }
 
-    pub fn ty(&self) -> Binder<Ty<'tcx>> {
+    pub fn ty(&self) -> Binder<'tcx, Ty<'tcx>> {
         self.map_bound(|predicate| predicate.ty)
     }
 
@@ -1346,7 +694,7 @@ pub trait ToPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx>;
 }
 
-impl ToPredicate<'tcx> for Binder<PredicateKind<'tcx>> {
+impl ToPredicate<'tcx> for Binder<'tcx, PredicateKind<'tcx>> {
     #[inline(always)]
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
         tcx.mk_predicate(self)
@@ -1369,11 +717,11 @@ impl<'tcx> ToPredicate<'tcx> for ConstnessAnd<TraitRef<'tcx>> {
 
 impl<'tcx> ToPredicate<'tcx> for ConstnessAnd<PolyTraitRef<'tcx>> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
-        ConstnessAnd {
-            value: self.value.map_bound(|trait_ref| ty::TraitPredicate { trait_ref }),
-            constness: self.constness,
-        }
-        .to_predicate(tcx)
+        self.value
+            .map_bound(|trait_ref| {
+                PredicateKind::Trait(ty::TraitPredicate { trait_ref }, self.constness)
+            })
+            .to_predicate(tcx)
     }
 }
 
@@ -1704,10 +1052,6 @@ impl WithOptConstParam<DefId> {
         None
     }
 
-    pub fn expect_local(self) -> WithOptConstParam<LocalDefId> {
-        self.as_local().unwrap()
-    }
-
     pub fn is_local(self) -> bool {
         self.did.is_local()
     }
@@ -1932,32 +1276,6 @@ pub struct Destructor {
 
 bitflags! {
     #[derive(HashStable)]
-    pub struct AdtFlags: u32 {
-        const NO_ADT_FLAGS        = 0;
-        /// Indicates whether the ADT is an enum.
-        const IS_ENUM             = 1 << 0;
-        /// Indicates whether the ADT is a union.
-        const IS_UNION            = 1 << 1;
-        /// Indicates whether the ADT is a struct.
-        const IS_STRUCT           = 1 << 2;
-        /// Indicates whether the ADT is a struct and has a constructor.
-        const HAS_CTOR            = 1 << 3;
-        /// Indicates whether the type is `PhantomData`.
-        const IS_PHANTOM_DATA     = 1 << 4;
-        /// Indicates whether the type has a `#[fundamental]` attribute.
-        const IS_FUNDAMENTAL      = 1 << 5;
-        /// Indicates whether the type is `Box`.
-        const IS_BOX              = 1 << 6;
-        /// Indicates whether the type is `ManuallyDrop`.
-        const IS_MANUALLY_DROP    = 1 << 7;
-        /// Indicates whether the variant list of this ADT is `#[non_exhaustive]`.
-        /// (i.e., this flag is never set unless this ADT is an enum).
-        const IS_VARIANT_LIST_NON_EXHAUSTIVE = 1 << 8;
-    }
-}
-
-bitflags! {
-    #[derive(HashStable)]
     pub struct VariantFlags: u32 {
         const NO_VARIANT_FLAGS        = 0;
         /// Indicates whether the field list of this variant is `#[non_exhaustive]`.
@@ -2079,105 +1397,6 @@ pub struct FieldDef {
     pub vis: Visibility,
 }
 
-/// The definition of a user-defined type, e.g., a `struct`, `enum`, or `union`.
-///
-/// These are all interned (by `alloc_adt_def`) into the global arena.
-///
-/// The initialism *ADT* stands for an [*algebraic data type (ADT)*][adt].
-/// This is slightly wrong because `union`s are not ADTs.
-/// Moreover, Rust only allows recursive data types through indirection.
-///
-/// [adt]: https://en.wikipedia.org/wiki/Algebraic_data_type
-pub struct AdtDef {
-    /// The `DefId` of the struct, enum or union item.
-    pub did: DefId,
-    /// Variants of the ADT. If this is a struct or union, then there will be a single variant.
-    pub variants: IndexVec<VariantIdx, VariantDef>,
-    /// Flags of the ADT (e.g., is this a struct? is this non-exhaustive?).
-    flags: AdtFlags,
-    /// Repr options provided by the user.
-    pub repr: ReprOptions,
-}
-
-impl PartialOrd for AdtDef {
-    fn partial_cmp(&self, other: &AdtDef) -> Option<Ordering> {
-        Some(self.cmp(&other))
-    }
-}
-
-/// There should be only one AdtDef for each `did`, therefore
-/// it is fine to implement `Ord` only based on `did`.
-impl Ord for AdtDef {
-    fn cmp(&self, other: &AdtDef) -> Ordering {
-        self.did.cmp(&other.did)
-    }
-}
-
-impl PartialEq for AdtDef {
-    // `AdtDef`s are always interned, and this is part of `TyS` equality.
-    #[inline]
-    fn eq(&self, other: &Self) -> bool {
-        ptr::eq(self, other)
-    }
-}
-
-impl Eq for AdtDef {}
-
-impl Hash for AdtDef {
-    #[inline]
-    fn hash<H: Hasher>(&self, s: &mut H) {
-        (self as *const AdtDef).hash(s)
-    }
-}
-
-impl<S: Encoder> Encodable<S> for AdtDef {
-    fn encode(&self, s: &mut S) -> Result<(), S::Error> {
-        self.did.encode(s)
-    }
-}
-
-impl<'a> HashStable<StableHashingContext<'a>> for AdtDef {
-    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        thread_local! {
-            static CACHE: RefCell<FxHashMap<usize, Fingerprint>> = Default::default();
-        }
-
-        let hash: Fingerprint = CACHE.with(|cache| {
-            let addr = self as *const AdtDef as usize;
-            *cache.borrow_mut().entry(addr).or_insert_with(|| {
-                let ty::AdtDef { did, ref variants, ref flags, ref repr } = *self;
-
-                let mut hasher = StableHasher::new();
-                did.hash_stable(hcx, &mut hasher);
-                variants.hash_stable(hcx, &mut hasher);
-                flags.hash_stable(hcx, &mut hasher);
-                repr.hash_stable(hcx, &mut hasher);
-
-                hasher.finish()
-            })
-        });
-
-        hash.hash_stable(hcx, hasher);
-    }
-}
-
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
-pub enum AdtKind {
-    Struct,
-    Union,
-    Enum,
-}
-
-impl Into<DataTypeKind> for AdtKind {
-    fn into(self) -> DataTypeKind {
-        match self {
-            AdtKind::Struct => DataTypeKind::Struct,
-            AdtKind::Union => DataTypeKind::Union,
-            AdtKind::Enum => DataTypeKind::Enum,
-        }
-    }
-}
-
 bitflags! {
     #[derive(TyEncodable, TyDecodable, Default, HashStable)]
     pub struct ReprFlags: u8 {
@@ -2300,334 +1519,6 @@ impl ReprOptions {
     }
 }
 
-impl<'tcx> AdtDef {
-    /// Creates a new `AdtDef`.
-    fn new(
-        tcx: TyCtxt<'_>,
-        did: DefId,
-        kind: AdtKind,
-        variants: IndexVec<VariantIdx, VariantDef>,
-        repr: ReprOptions,
-    ) -> Self {
-        debug!("AdtDef::new({:?}, {:?}, {:?}, {:?})", did, kind, variants, repr);
-        let mut flags = AdtFlags::NO_ADT_FLAGS;
-
-        if kind == AdtKind::Enum && tcx.has_attr(did, sym::non_exhaustive) {
-            debug!("found non-exhaustive variant list for {:?}", did);
-            flags = flags | AdtFlags::IS_VARIANT_LIST_NON_EXHAUSTIVE;
-        }
-
-        flags |= match kind {
-            AdtKind::Enum => AdtFlags::IS_ENUM,
-            AdtKind::Union => AdtFlags::IS_UNION,
-            AdtKind::Struct => AdtFlags::IS_STRUCT,
-        };
-
-        if kind == AdtKind::Struct && variants[VariantIdx::new(0)].ctor_def_id.is_some() {
-            flags |= AdtFlags::HAS_CTOR;
-        }
-
-        let attrs = tcx.get_attrs(did);
-        if tcx.sess.contains_name(&attrs, sym::fundamental) {
-            flags |= AdtFlags::IS_FUNDAMENTAL;
-        }
-        if Some(did) == tcx.lang_items().phantom_data() {
-            flags |= AdtFlags::IS_PHANTOM_DATA;
-        }
-        if Some(did) == tcx.lang_items().owned_box() {
-            flags |= AdtFlags::IS_BOX;
-        }
-        if Some(did) == tcx.lang_items().manually_drop() {
-            flags |= AdtFlags::IS_MANUALLY_DROP;
-        }
-
-        AdtDef { did, variants, flags, repr }
-    }
-
-    /// Returns `true` if this is a struct.
-    #[inline]
-    pub fn is_struct(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_STRUCT)
-    }
-
-    /// Returns `true` if this is a union.
-    #[inline]
-    pub fn is_union(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_UNION)
-    }
-
-    /// Returns `true` if this is a enum.
-    #[inline]
-    pub fn is_enum(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_ENUM)
-    }
-
-    /// Returns `true` if the variant list of this ADT is `#[non_exhaustive]`.
-    #[inline]
-    pub fn is_variant_list_non_exhaustive(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_VARIANT_LIST_NON_EXHAUSTIVE)
-    }
-
-    /// Returns the kind of the ADT.
-    #[inline]
-    pub fn adt_kind(&self) -> AdtKind {
-        if self.is_enum() {
-            AdtKind::Enum
-        } else if self.is_union() {
-            AdtKind::Union
-        } else {
-            AdtKind::Struct
-        }
-    }
-
-    /// Returns a description of this abstract data type.
-    pub fn descr(&self) -> &'static str {
-        match self.adt_kind() {
-            AdtKind::Struct => "struct",
-            AdtKind::Union => "union",
-            AdtKind::Enum => "enum",
-        }
-    }
-
-    /// Returns a description of a variant of this abstract data type.
-    #[inline]
-    pub fn variant_descr(&self) -> &'static str {
-        match self.adt_kind() {
-            AdtKind::Struct => "struct",
-            AdtKind::Union => "union",
-            AdtKind::Enum => "variant",
-        }
-    }
-
-    /// If this function returns `true`, it implies that `is_struct` must return `true`.
-    #[inline]
-    pub fn has_ctor(&self) -> bool {
-        self.flags.contains(AdtFlags::HAS_CTOR)
-    }
-
-    /// Returns `true` if this type is `#[fundamental]` for the purposes
-    /// of coherence checking.
-    #[inline]
-    pub fn is_fundamental(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_FUNDAMENTAL)
-    }
-
-    /// Returns `true` if this is `PhantomData<T>`.
-    #[inline]
-    pub fn is_phantom_data(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_PHANTOM_DATA)
-    }
-
-    /// Returns `true` if this is Box<T>.
-    #[inline]
-    pub fn is_box(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_BOX)
-    }
-
-    /// Returns `true` if this is `ManuallyDrop<T>`.
-    #[inline]
-    pub fn is_manually_drop(&self) -> bool {
-        self.flags.contains(AdtFlags::IS_MANUALLY_DROP)
-    }
-
-    /// Returns `true` if this type has a destructor.
-    pub fn has_dtor(&self, tcx: TyCtxt<'tcx>) -> bool {
-        self.destructor(tcx).is_some()
-    }
-
-    /// Asserts this is a struct or union and returns its unique variant.
-    pub fn non_enum_variant(&self) -> &VariantDef {
-        assert!(self.is_struct() || self.is_union());
-        &self.variants[VariantIdx::new(0)]
-    }
-
-    #[inline]
-    pub fn predicates(&self, tcx: TyCtxt<'tcx>) -> GenericPredicates<'tcx> {
-        tcx.predicates_of(self.did)
-    }
-
-    /// Returns an iterator over all fields contained
-    /// by this ADT.
-    #[inline]
-    pub fn all_fields(&self) -> impl Iterator<Item = &FieldDef> + Clone {
-        self.variants.iter().flat_map(|v| v.fields.iter())
-    }
-
-    /// Whether the ADT lacks fields. Note that this includes uninhabited enums,
-    /// e.g., `enum Void {}` is considered payload free as well.
-    pub fn is_payloadfree(&self) -> bool {
-        self.variants.iter().all(|v| v.fields.is_empty())
-    }
-
-    /// Return a `VariantDef` given a variant id.
-    pub fn variant_with_id(&self, vid: DefId) -> &VariantDef {
-        self.variants.iter().find(|v| v.def_id == vid).expect("variant_with_id: unknown variant")
-    }
-
-    /// Return a `VariantDef` given a constructor id.
-    pub fn variant_with_ctor_id(&self, cid: DefId) -> &VariantDef {
-        self.variants
-            .iter()
-            .find(|v| v.ctor_def_id == Some(cid))
-            .expect("variant_with_ctor_id: unknown variant")
-    }
-
-    /// Return the index of `VariantDef` given a variant id.
-    pub fn variant_index_with_id(&self, vid: DefId) -> VariantIdx {
-        self.variants
-            .iter_enumerated()
-            .find(|(_, v)| v.def_id == vid)
-            .expect("variant_index_with_id: unknown variant")
-            .0
-    }
-
-    /// Return the index of `VariantDef` given a constructor id.
-    pub fn variant_index_with_ctor_id(&self, cid: DefId) -> VariantIdx {
-        self.variants
-            .iter_enumerated()
-            .find(|(_, v)| v.ctor_def_id == Some(cid))
-            .expect("variant_index_with_ctor_id: unknown variant")
-            .0
-    }
-
-    pub fn variant_of_res(&self, res: Res) -> &VariantDef {
-        match res {
-            Res::Def(DefKind::Variant, vid) => self.variant_with_id(vid),
-            Res::Def(DefKind::Ctor(..), cid) => self.variant_with_ctor_id(cid),
-            Res::Def(DefKind::Struct, _)
-            | Res::Def(DefKind::Union, _)
-            | Res::Def(DefKind::TyAlias, _)
-            | Res::Def(DefKind::AssocTy, _)
-            | Res::SelfTy(..)
-            | Res::SelfCtor(..) => self.non_enum_variant(),
-            _ => bug!("unexpected res {:?} in variant_of_res", res),
-        }
-    }
-
-    #[inline]
-    pub fn eval_explicit_discr(&self, tcx: TyCtxt<'tcx>, expr_did: DefId) -> Option<Discr<'tcx>> {
-        assert!(self.is_enum());
-        let param_env = tcx.param_env(expr_did);
-        let repr_type = self.repr.discr_type();
-        match tcx.const_eval_poly(expr_did) {
-            Ok(val) => {
-                let ty = repr_type.to_ty(tcx);
-                if let Some(b) = val.try_to_bits_for_ty(tcx, param_env, ty) {
-                    trace!("discriminants: {} ({:?})", b, repr_type);
-                    Some(Discr { val: b, ty })
-                } else {
-                    info!("invalid enum discriminant: {:#?}", val);
-                    crate::mir::interpret::struct_error(
-                        tcx.at(tcx.def_span(expr_did)),
-                        "constant evaluation of enum discriminant resulted in non-integer",
-                    )
-                    .emit();
-                    None
-                }
-            }
-            Err(err) => {
-                let msg = match err {
-                    ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => {
-                        "enum discriminant evaluation failed"
-                    }
-                    ErrorHandled::TooGeneric => "enum discriminant depends on generics",
-                };
-                tcx.sess.delay_span_bug(tcx.def_span(expr_did), msg);
-                None
-            }
-        }
-    }
-
-    #[inline]
-    pub fn discriminants(
-        &'tcx self,
-        tcx: TyCtxt<'tcx>,
-    ) -> impl Iterator<Item = (VariantIdx, Discr<'tcx>)> + Captures<'tcx> {
-        assert!(self.is_enum());
-        let repr_type = self.repr.discr_type();
-        let initial = repr_type.initial_discriminant(tcx);
-        let mut prev_discr = None::<Discr<'tcx>>;
-        self.variants.iter_enumerated().map(move |(i, v)| {
-            let mut discr = prev_discr.map_or(initial, |d| d.wrap_incr(tcx));
-            if let VariantDiscr::Explicit(expr_did) = v.discr {
-                if let Some(new_discr) = self.eval_explicit_discr(tcx, expr_did) {
-                    discr = new_discr;
-                }
-            }
-            prev_discr = Some(discr);
-
-            (i, discr)
-        })
-    }
-
-    #[inline]
-    pub fn variant_range(&self) -> Range<VariantIdx> {
-        VariantIdx::new(0)..VariantIdx::new(self.variants.len())
-    }
-
-    /// Computes the discriminant value used by a specific variant.
-    /// Unlike `discriminants`, this is (amortized) constant-time,
-    /// only doing at most one query for evaluating an explicit
-    /// discriminant (the last one before the requested variant),
-    /// assuming there are no constant-evaluation errors there.
-    #[inline]
-    pub fn discriminant_for_variant(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        variant_index: VariantIdx,
-    ) -> Discr<'tcx> {
-        assert!(self.is_enum());
-        let (val, offset) = self.discriminant_def_for_variant(variant_index);
-        let explicit_value = val
-            .and_then(|expr_did| self.eval_explicit_discr(tcx, expr_did))
-            .unwrap_or_else(|| self.repr.discr_type().initial_discriminant(tcx));
-        explicit_value.checked_add(tcx, offset as u128).0
-    }
-
-    /// Yields a `DefId` for the discriminant and an offset to add to it
-    /// Alternatively, if there is no explicit discriminant, returns the
-    /// inferred discriminant directly.
-    pub fn discriminant_def_for_variant(&self, variant_index: VariantIdx) -> (Option<DefId>, u32) {
-        assert!(!self.variants.is_empty());
-        let mut explicit_index = variant_index.as_u32();
-        let expr_did;
-        loop {
-            match self.variants[VariantIdx::from_u32(explicit_index)].discr {
-                ty::VariantDiscr::Relative(0) => {
-                    expr_did = None;
-                    break;
-                }
-                ty::VariantDiscr::Relative(distance) => {
-                    explicit_index -= distance;
-                }
-                ty::VariantDiscr::Explicit(did) => {
-                    expr_did = Some(did);
-                    break;
-                }
-            }
-        }
-        (expr_did, variant_index.as_u32() - explicit_index)
-    }
-
-    pub fn destructor(&self, tcx: TyCtxt<'tcx>) -> Option<Destructor> {
-        tcx.adt_destructor(self.did)
-    }
-
-    /// Returns a list of types such that `Self: Sized` if and only
-    /// if that type is `Sized`, or `TyErr` if this type is recursive.
-    ///
-    /// Oddly enough, checking that the sized-constraint is `Sized` is
-    /// actually more expressive than checking all members:
-    /// the `Sized` trait is inductive, so an associated type that references
-    /// `Self` would prevent its containing ADT from being `Sized`.
-    ///
-    /// Due to normalization being eager, this applies even if
-    /// the associated type is behind a pointer (e.g., issue #31299).
-    pub fn sized_constraint(&self, tcx: TyCtxt<'tcx>) -> &'tcx [Ty<'tcx>] {
-        tcx.adt_sized_constraint(self.did).0
-    }
-}
-
 impl<'tcx> FieldDef {
     /// Returns the type of this field. The `subst` is typically obtained
     /// via the second field of `TyKind::AdtDef`.
@@ -2636,93 +1527,6 @@ impl<'tcx> FieldDef {
     }
 }
 
-/// Represents the various closure traits in the language. This
-/// will determine the type of the environment (`self`, in the
-/// desugaring) argument that the closure expects.
-///
-/// You can get the environment type of a closure using
-/// `tcx.closure_env_ty()`.
-#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable)]
-pub enum ClosureKind {
-    // Warning: Ordering is significant here! The ordering is chosen
-    // because the trait Fn is a subtrait of FnMut and so in turn, and
-    // hence we order it so that Fn < FnMut < FnOnce.
-    Fn,
-    FnMut,
-    FnOnce,
-}
-
-impl<'tcx> ClosureKind {
-    // This is the initial value used when doing upvar inference.
-    pub const LATTICE_BOTTOM: ClosureKind = ClosureKind::Fn;
-
-    pub fn trait_did(&self, tcx: TyCtxt<'tcx>) -> DefId {
-        match *self {
-            ClosureKind::Fn => tcx.require_lang_item(LangItem::Fn, None),
-            ClosureKind::FnMut => tcx.require_lang_item(LangItem::FnMut, None),
-            ClosureKind::FnOnce => tcx.require_lang_item(LangItem::FnOnce, None),
-        }
-    }
-
-    /// Returns `true` if a type that impls this closure kind
-    /// must also implement `other`.
-    pub fn extends(self, other: ty::ClosureKind) -> bool {
-        matches!(
-            (self, other),
-            (ClosureKind::Fn, ClosureKind::Fn)
-                | (ClosureKind::Fn, ClosureKind::FnMut)
-                | (ClosureKind::Fn, ClosureKind::FnOnce)
-                | (ClosureKind::FnMut, ClosureKind::FnMut)
-                | (ClosureKind::FnMut, ClosureKind::FnOnce)
-                | (ClosureKind::FnOnce, ClosureKind::FnOnce)
-        )
-    }
-
-    /// Returns the representative scalar type for this closure kind.
-    /// See `TyS::to_opt_closure_kind` for more details.
-    pub fn to_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
-        match self {
-            ty::ClosureKind::Fn => tcx.types.i8,
-            ty::ClosureKind::FnMut => tcx.types.i16,
-            ty::ClosureKind::FnOnce => tcx.types.i32,
-        }
-    }
-}
-
-impl BorrowKind {
-    pub fn from_mutbl(m: hir::Mutability) -> BorrowKind {
-        match m {
-            hir::Mutability::Mut => MutBorrow,
-            hir::Mutability::Not => ImmBorrow,
-        }
-    }
-
-    /// Returns a mutability `m` such that an `&m T` pointer could be used to obtain this borrow
-    /// kind. Because borrow kinds are richer than mutabilities, we sometimes have to pick a
-    /// mutability that is stronger than necessary so that it at least *would permit* the borrow in
-    /// question.
-    pub fn to_mutbl_lossy(self) -> hir::Mutability {
-        match self {
-            MutBorrow => hir::Mutability::Mut,
-            ImmBorrow => hir::Mutability::Not,
-
-            // We have no type corresponding to a unique imm borrow, so
-            // use `&mut`. It gives all the capabilities of an `&uniq`
-            // and hence is a safe "over approximation".
-            UniqueImmBorrow => hir::Mutability::Mut,
-        }
-    }
-
-    pub fn to_user_str(&self) -> &'static str {
-        match *self {
-            MutBorrow => "mutable",
-            ImmBorrow => "immutable",
-            UniqueImmBorrow => "uniquely immutable",
-        }
-    }
-}
-
 pub type Attributes<'tcx> = &'tcx [ast::Attribute];
 
 #[derive(Debug, PartialEq, Eq)]
@@ -2963,7 +1767,10 @@ impl<'tcx> TyCtxt<'tcx> {
                 | DefKind::AnonConst => self.mir_for_ctfe_opt_const_arg(def),
                 // If the caller wants `mir_for_ctfe` of a function they should not be using
                 // `instance_mir`, so we'll assume const fn also wants the optimized version.
-                _ => self.optimized_mir_or_const_arg_mir(def),
+                _ => {
+                    assert_eq!(def.const_param_did, None);
+                    self.optimized_mir(def.did)
+                }
             },
             ty::InstanceDef::VtableShim(..)
             | ty::InstanceDef::ReifyShim(..)
@@ -3077,9 +1884,6 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 }
 
-#[derive(Clone, HashStable, Debug)]
-pub struct AdtSizedConstraint<'tcx>(pub &'tcx [Ty<'tcx>]);
-
 /// Yields the parent function's `DefId` if `def_id` is an `impl Trait` definition.
 pub fn is_impl_trait_defn(tcx: TyCtxt<'_>, def_id: DefId) -> Option<DefId> {
     if let Some(def_id) = def_id.as_local() {
@@ -3154,6 +1958,7 @@ pub fn provide(providers: &mut ty::query::Providers) {
         trait_impls_of: trait_def::trait_impls_of_provider,
         all_local_trait_impls: trait_def::all_local_trait_impls,
         type_uninhabited_from: inhabitedness::type_uninhabited_from,
+        const_param_default: consts::const_param_default,
         ..*providers
     };
 }
diff --git a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
index 9d97815a5f1..a4f736654af 100644
--- a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
+++ b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
@@ -7,6 +7,7 @@
 //! `normalize_generic_arg_after_erasing_regions` query for each type
 //! or constant found within. (This underlying query is what is cached.)
 
+use crate::mir;
 use crate::ty::fold::{TypeFoldable, TypeFolder};
 use crate::ty::subst::{Subst, SubstsRef};
 use crate::ty::{self, Ty, TyCtxt};
@@ -38,7 +39,7 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    /// If you have a `Binder<T>`, you can do this to strip out the
+    /// If you have a `Binder<'tcx, T>`, you can do this to strip out the
     /// late-bound regions and then normalize the result, yielding up
     /// a `T` (with regions erased). This is appropriate when the
     /// binder is being instantiated at the call site.
@@ -49,7 +50,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn normalize_erasing_late_bound_regions<T>(
         self,
         param_env: ty::ParamEnv<'tcx>,
-        value: ty::Binder<T>,
+        value: ty::Binder<'tcx, T>,
     ) -> T
     where
         T: TypeFoldable<'tcx>,
@@ -101,4 +102,10 @@ impl TypeFolder<'tcx> for NormalizeAfterErasingRegionsFolder<'tcx> {
         let arg = self.param_env.and(c.into());
         self.tcx.normalize_generic_arg_after_erasing_regions(arg).expect_const()
     }
+
+    #[inline]
+    fn fold_mir_const(&mut self, c: mir::ConstantKind<'tcx>) -> mir::ConstantKind<'tcx> {
+        let arg = self.param_env.and(c);
+        self.tcx.normalize_mir_const_after_erasing_regions(arg)
+    }
 }
diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index 77f16688937..13e2122a619 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -63,7 +63,7 @@ pub trait Printer<'tcx>: Sized {
 
     fn print_dyn_existential(
         self,
-        predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     ) -> Result<Self::DynExistential, Self::Error>;
 
     fn print_const(self, ct: &'tcx ty::Const<'tcx>) -> Result<Self::Const, Self::Error>;
@@ -193,17 +193,19 @@ pub trait Printer<'tcx>: Sized {
             .params
             .iter()
             .rev()
-            .take_while(|param| {
-                match param.kind {
-                    ty::GenericParamDefKind::Lifetime => false,
-                    ty::GenericParamDefKind::Type { has_default, .. } => {
-                        has_default
-                            && substs[param.index as usize]
-                                == GenericArg::from(
-                                    self.tcx().type_of(param.def_id).subst(self.tcx(), substs),
-                                )
-                    }
-                    ty::GenericParamDefKind::Const => false, // FIXME(const_generics_defaults)
+            .take_while(|param| match param.kind {
+                ty::GenericParamDefKind::Lifetime => false,
+                ty::GenericParamDefKind::Type { has_default, .. } => {
+                    has_default
+                        && substs[param.index as usize]
+                            == GenericArg::from(
+                                self.tcx().type_of(param.def_id).subst(self.tcx(), substs),
+                            )
+                }
+                ty::GenericParamDefKind::Const { has_default } => {
+                    has_default
+                        && substs[param.index as usize]
+                            == GenericArg::from(self.tcx().const_param_default(param.def_id))
                 }
             })
             .count();
@@ -344,7 +346,7 @@ impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for Ty<'tcx> {
 }
 
 impl<'tcx, P: Printer<'tcx>> Print<'tcx, P>
-    for &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>
+    for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>
 {
     type Output = P::DynExistential;
     type Error = P::Error;
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 53c164d44b3..1989c91a879 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -4,6 +4,7 @@ use crate::ty::subst::{GenericArg, GenericArgKind, Subst};
 use crate::ty::{self, ConstInt, DefIdTree, ParamConst, ScalarInt, Ty, TyCtxt, TypeFoldable};
 use rustc_apfloat::ieee::{Double, Single};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sso::SsoHashSet;
 use rustc_hir as hir;
 use rustc_hir::def::{self, CtorKind, DefKind, Namespace};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdSet, CRATE_DEF_INDEX, LOCAL_CRATE};
@@ -19,6 +20,7 @@ use std::char;
 use std::collections::BTreeMap;
 use std::convert::TryFrom;
 use std::fmt::{self, Write as _};
+use std::iter;
 use std::ops::{ControlFlow, Deref, DerefMut};
 
 // `pretty` is a separate module only for organization.
@@ -201,7 +203,7 @@ pub trait PrettyPrinter<'tcx>:
         self.print_def_path(def_id, substs)
     }
 
-    fn in_binder<T>(self, value: &ty::Binder<T>) -> Result<Self, Self::Error>
+    fn in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, Self::Error>
     where
         T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<'tcx>,
     {
@@ -210,7 +212,7 @@ pub trait PrettyPrinter<'tcx>:
 
     fn wrap_binder<T, F: Fn(&T, Self) -> Result<Self, fmt::Error>>(
         self,
-        value: &ty::Binder<T>,
+        value: &ty::Binder<'tcx, T>,
         f: F,
     ) -> Result<Self, Self::Error>
     where
@@ -764,7 +766,7 @@ pub trait PrettyPrinter<'tcx>:
 
     fn pretty_print_dyn_existential(
         mut self,
-        predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     ) -> Result<Self::DynExistential, Self::Error> {
         // Generate the main trait ref, including associated types.
         let mut first = true;
@@ -915,7 +917,7 @@ pub trait PrettyPrinter<'tcx>:
         }
 
         match ct.val {
-            ty::ConstKind::Unevaluated(def, substs, promoted) => {
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) => {
                 if let Some(promoted) = promoted {
                     p!(print_value_path(def.did, substs));
                     p!(write("::{:?}", promoted));
@@ -956,32 +958,40 @@ pub trait PrettyPrinter<'tcx>:
     }
 
     fn pretty_print_const_scalar(
-        mut self,
+        self,
         scalar: Scalar,
         ty: Ty<'tcx>,
         print_ty: bool,
     ) -> Result<Self::Const, Self::Error> {
+        match scalar {
+            Scalar::Ptr(ptr) => self.pretty_print_const_scalar_ptr(ptr, ty, print_ty),
+            Scalar::Int(int) => self.pretty_print_const_scalar_int(int, ty, print_ty),
+        }
+    }
+
+    fn pretty_print_const_scalar_ptr(
+        mut self,
+        ptr: Pointer,
+        ty: Ty<'tcx>,
+        print_ty: bool,
+    ) -> Result<Self::Const, Self::Error> {
         define_scoped_cx!(self);
 
-        match (scalar, &ty.kind()) {
+        match ty.kind() {
             // Byte strings (&[u8; N])
-            (
-                Scalar::Ptr(ptr),
-                ty::Ref(
-                    _,
-                    ty::TyS {
-                        kind:
-                            ty::Array(
-                                ty::TyS { kind: ty::Uint(ty::UintTy::U8), .. },
-                                ty::Const {
-                                    val: ty::ConstKind::Value(ConstValue::Scalar(int)),
-                                    ..
-                                },
-                            ),
-                        ..
-                    },
-                    _,
-                ),
+            ty::Ref(
+                _,
+                ty::TyS {
+                    kind:
+                        ty::Array(
+                            ty::TyS { kind: ty::Uint(ty::UintTy::U8), .. },
+                            ty::Const {
+                                val: ty::ConstKind::Value(ConstValue::Scalar(int)), ..
+                            },
+                        ),
+                    ..
+                },
+                _,
             ) => match self.tcx().get_global_alloc(ptr.alloc_id) {
                 Some(GlobalAlloc::Memory(alloc)) => {
                     let bytes = int.assert_bits(self.tcx().data_layout.pointer_size);
@@ -997,28 +1007,59 @@ pub trait PrettyPrinter<'tcx>:
                 Some(GlobalAlloc::Function(_)) => p!("<function>"),
                 None => p!("<dangling pointer>"),
             },
+            ty::FnPtr(_) => {
+                // FIXME: We should probably have a helper method to share code with the "Byte strings"
+                // printing above (which also has to handle pointers to all sorts of things).
+                match self.tcx().get_global_alloc(ptr.alloc_id) {
+                    Some(GlobalAlloc::Function(instance)) => {
+                        self = self.typed_value(
+                            |this| this.print_value_path(instance.def_id(), instance.substs),
+                            |this| this.print_type(ty),
+                            " as ",
+                        )?;
+                    }
+                    _ => self = self.pretty_print_const_pointer(ptr, ty, print_ty)?,
+                }
+            }
+            // Any pointer values not covered by a branch above
+            _ => {
+                self = self.pretty_print_const_pointer(ptr, ty, print_ty)?;
+            }
+        }
+        Ok(self)
+    }
+
+    fn pretty_print_const_scalar_int(
+        mut self,
+        int: ScalarInt,
+        ty: Ty<'tcx>,
+        print_ty: bool,
+    ) -> Result<Self::Const, Self::Error> {
+        define_scoped_cx!(self);
+
+        match ty.kind() {
             // Bool
-            (Scalar::Int(int), ty::Bool) if int == ScalarInt::FALSE => p!("false"),
-            (Scalar::Int(int), ty::Bool) if int == ScalarInt::TRUE => p!("true"),
+            ty::Bool if int == ScalarInt::FALSE => p!("false"),
+            ty::Bool if int == ScalarInt::TRUE => p!("true"),
             // Float
-            (Scalar::Int(int), ty::Float(ty::FloatTy::F32)) => {
+            ty::Float(ty::FloatTy::F32) => {
                 p!(write("{}f32", Single::try_from(int).unwrap()))
             }
-            (Scalar::Int(int), ty::Float(ty::FloatTy::F64)) => {
+            ty::Float(ty::FloatTy::F64) => {
                 p!(write("{}f64", Double::try_from(int).unwrap()))
             }
             // Int
-            (Scalar::Int(int), ty::Uint(_) | ty::Int(_)) => {
+            ty::Uint(_) | ty::Int(_) => {
                 let int =
                     ConstInt::new(int, matches!(ty.kind(), ty::Int(_)), ty.is_ptr_sized_integral());
                 if print_ty { p!(write("{:#?}", int)) } else { p!(write("{:?}", int)) }
             }
             // Char
-            (Scalar::Int(int), ty::Char) if char::try_from(int).is_ok() => {
+            ty::Char if char::try_from(int).is_ok() => {
                 p!(write("{:?}", char::try_from(int).unwrap()))
             }
             // Raw pointers
-            (Scalar::Int(int), ty::RawPtr(_) | ty::FnPtr(_)) => {
+            ty::RawPtr(_) | ty::FnPtr(_) => {
                 let data = int.assert_bits(self.tcx().data_layout.pointer_size);
                 self = self.typed_value(
                     |mut this| {
@@ -1029,26 +1070,12 @@ pub trait PrettyPrinter<'tcx>:
                     " as ",
                 )?;
             }
-            (Scalar::Ptr(ptr), ty::FnPtr(_)) => {
-                // FIXME: We should probably have a helper method to share code with the "Byte strings"
-                // printing above (which also has to handle pointers to all sorts of things).
-                match self.tcx().get_global_alloc(ptr.alloc_id) {
-                    Some(GlobalAlloc::Function(instance)) => {
-                        self = self.typed_value(
-                            |this| this.print_value_path(instance.def_id(), instance.substs),
-                            |this| this.print_type(ty),
-                            " as ",
-                        )?;
-                    }
-                    _ => self = self.pretty_print_const_pointer(ptr, ty, print_ty)?,
-                }
-            }
             // For function type zsts just printing the path is enough
-            (Scalar::Int(int), ty::FnDef(d, s)) if int == ScalarInt::ZST => {
+            ty::FnDef(d, s) if int == ScalarInt::ZST => {
                 p!(print_value_path(*d, s))
             }
             // Nontrivial types with scalar bit representation
-            (Scalar::Int(int), _) => {
+            _ => {
                 let print = |mut this: Self| {
                     if int.size() == Size::ZERO {
                         write!(this, "transmute(())")?;
@@ -1063,10 +1090,6 @@ pub trait PrettyPrinter<'tcx>:
                     print(self)?
                 };
             }
-            // Any pointer values not covered by a branch above
-            (Scalar::Ptr(p), _) => {
-                self = self.pretty_print_const_pointer(p, ty, print_ty)?;
-            }
         }
         Ok(self)
     }
@@ -1202,7 +1225,7 @@ pub trait PrettyPrinter<'tcx>:
                             CtorKind::Fictive => {
                                 p!(" {{ ");
                                 let mut first = true;
-                                for (field_def, field) in variant_def.fields.iter().zip(fields) {
+                                for (field_def, field) in iter::zip(&variant_def.fields, fields) {
                                     if !first {
                                         p!(", ");
                                     }
@@ -1399,7 +1422,8 @@ impl<F: fmt::Write> Printer<'tcx> for FmtPrinter<'_, 'tcx, F> {
     }
 
     fn print_type(mut self, ty: Ty<'tcx>) -> Result<Self::Type, Self::Error> {
-        if self.tcx.sess.type_length_limit().value_within_limit(self.printed_type_count) {
+        let type_length_limit = self.tcx.sess.type_length_limit();
+        if type_length_limit.value_within_limit(self.printed_type_count) {
             self.printed_type_count += 1;
             self.pretty_print_type(ty)
         } else {
@@ -1410,7 +1434,7 @@ impl<F: fmt::Write> Printer<'tcx> for FmtPrinter<'_, 'tcx, F> {
 
     fn print_dyn_existential(
         self,
-        predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     ) -> Result<Self::DynExistential, Self::Error> {
         self.pretty_print_dyn_existential(predicates)
     }
@@ -1549,7 +1573,7 @@ impl<F: fmt::Write> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx, F> {
         Ok(self)
     }
 
-    fn in_binder<T>(self, value: &ty::Binder<T>) -> Result<Self, Self::Error>
+    fn in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, Self::Error>
     where
         T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<'tcx>,
     {
@@ -1558,7 +1582,7 @@ impl<F: fmt::Write> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx, F> {
 
     fn wrap_binder<T, C: Fn(&T, Self) -> Result<Self, Self::Error>>(
         self,
-        value: &ty::Binder<T>,
+        value: &ty::Binder<'tcx, T>,
         f: C,
     ) -> Result<Self, Self::Error>
     where
@@ -1614,7 +1638,7 @@ impl<F: fmt::Write> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx, F> {
                 data.name != kw::Empty && data.name != kw::UnderscoreLifetime
             }
 
-            ty::ReLateBound(_, ty::BoundRegion { kind: br })
+            ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
             | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
             | ty::RePlaceholder(ty::Placeholder { name: br, .. }) => {
                 if let ty::BrNamed(_, name) = br {
@@ -1693,7 +1717,7 @@ impl<F: fmt::Write> FmtPrinter<'_, '_, F> {
                     return Ok(self);
                 }
             }
-            ty::ReLateBound(_, ty::BoundRegion { kind: br })
+            ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
             | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
             | ty::RePlaceholder(ty::Placeholder { name: br, .. }) => {
                 if let ty::BrNamed(_, name) = br {
@@ -1741,7 +1765,7 @@ impl<F: fmt::Write> FmtPrinter<'_, '_, F> {
 impl<F: fmt::Write> FmtPrinter<'_, 'tcx, F> {
     pub fn name_all_regions<T>(
         mut self,
-        value: &ty::Binder<T>,
+        value: &ty::Binder<'tcx, T>,
     ) -> Result<(Self, (T, BTreeMap<ty::BoundRegion, ty::Region<'tcx>>)), fmt::Error>
     where
         T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<'tcx>,
@@ -1780,35 +1804,101 @@ impl<F: fmt::Write> FmtPrinter<'_, 'tcx, F> {
         define_scoped_cx!(self);
 
         let mut region_index = self.region_index;
-        let new_value = self.tcx.replace_late_bound_regions(value.clone(), |br| {
-            let _ = start_or_continue(&mut self, "for<", ", ");
-            let kind = match br.kind {
-                ty::BrNamed(_, name) => {
-                    let _ = write!(self, "{}", name);
-                    br.kind
-                }
-                ty::BrAnon(_) | ty::BrEnv => {
-                    let name = loop {
-                        let name = name_by_region_index(region_index);
-                        region_index += 1;
-                        if !self.used_region_names.contains(&name) {
-                            break name;
-                        }
-                    };
-                    let _ = write!(self, "{}", name);
-                    ty::BrNamed(DefId::local(CRATE_DEF_INDEX), name)
+        // If we want to print verbosly, then print *all* binders, even if they
+        // aren't named. Eventually, we might just want this as the default, but
+        // this is not *quite* right and changes the ordering of some output
+        // anyways.
+        let new_value = if self.tcx().sess.verbose() {
+            // anon index + 1 (BrEnv takes 0) -> name
+            let mut region_map: BTreeMap<u32, Symbol> = BTreeMap::default();
+            let bound_vars = value.bound_vars();
+            for var in bound_vars {
+                match var {
+                    ty::BoundVariableKind::Region(ty::BrNamed(_, name)) => {
+                        let _ = start_or_continue(&mut self, "for<", ", ");
+                        let _ = write!(self, "{}", name);
+                    }
+                    ty::BoundVariableKind::Region(ty::BrAnon(i)) => {
+                        let _ = start_or_continue(&mut self, "for<", ", ");
+                        let name = loop {
+                            let name = name_by_region_index(region_index);
+                            region_index += 1;
+                            if !self.used_region_names.contains(&name) {
+                                break name;
+                            }
+                        };
+                        let _ = write!(self, "{}", name);
+                        region_map.insert(i + 1, name);
+                    }
+                    ty::BoundVariableKind::Region(ty::BrEnv) => {
+                        let _ = start_or_continue(&mut self, "for<", ", ");
+                        let name = loop {
+                            let name = name_by_region_index(region_index);
+                            region_index += 1;
+                            if !self.used_region_names.contains(&name) {
+                                break name;
+                            }
+                        };
+                        let _ = write!(self, "{}", name);
+                        region_map.insert(0, name);
+                    }
+                    _ => continue,
                 }
-            };
-            self.tcx.mk_region(ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { kind }))
-        });
-        start_or_continue(&mut self, "", "> ")?;
+            }
+            start_or_continue(&mut self, "", "> ")?;
+
+            self.tcx.replace_late_bound_regions(value.clone(), |br| {
+                let kind = match br.kind {
+                    ty::BrNamed(_, _) => br.kind,
+                    ty::BrAnon(i) => {
+                        let name = region_map[&(i + 1)];
+                        ty::BrNamed(DefId::local(CRATE_DEF_INDEX), name)
+                    }
+                    ty::BrEnv => {
+                        let name = region_map[&0];
+                        ty::BrNamed(DefId::local(CRATE_DEF_INDEX), name)
+                    }
+                };
+                self.tcx.mk_region(ty::ReLateBound(
+                    ty::INNERMOST,
+                    ty::BoundRegion { var: br.var, kind },
+                ))
+            })
+        } else {
+            let new_value = self.tcx.replace_late_bound_regions(value.clone(), |br| {
+                let _ = start_or_continue(&mut self, "for<", ", ");
+                let kind = match br.kind {
+                    ty::BrNamed(_, name) => {
+                        let _ = write!(self, "{}", name);
+                        br.kind
+                    }
+                    ty::BrAnon(_) | ty::BrEnv => {
+                        let name = loop {
+                            let name = name_by_region_index(region_index);
+                            region_index += 1;
+                            if !self.used_region_names.contains(&name) {
+                                break name;
+                            }
+                        };
+                        let _ = write!(self, "{}", name);
+                        ty::BrNamed(DefId::local(CRATE_DEF_INDEX), name)
+                    }
+                };
+                self.tcx.mk_region(ty::ReLateBound(
+                    ty::INNERMOST,
+                    ty::BoundRegion { var: br.var, kind },
+                ))
+            });
+            start_or_continue(&mut self, "", "> ")?;
+            new_value
+        };
 
         self.binder_depth += 1;
         self.region_index = region_index;
         Ok((self, new_value))
     }
 
-    pub fn pretty_in_binder<T>(self, value: &ty::Binder<T>) -> Result<Self, fmt::Error>
+    pub fn pretty_in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, fmt::Error>
     where
         T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<'tcx>,
     {
@@ -1822,7 +1912,7 @@ impl<F: fmt::Write> FmtPrinter<'_, 'tcx, F> {
 
     pub fn pretty_wrap_binder<T, C: Fn(&T, Self) -> Result<Self, fmt::Error>>(
         self,
-        value: &ty::Binder<T>,
+        value: &ty::Binder<'tcx, T>,
         f: C,
     ) -> Result<Self, fmt::Error>
     where
@@ -1836,28 +1926,52 @@ impl<F: fmt::Write> FmtPrinter<'_, 'tcx, F> {
         Ok(inner)
     }
 
-    fn prepare_late_bound_region_info<T>(&mut self, value: &ty::Binder<T>)
+    fn prepare_late_bound_region_info<T>(&mut self, value: &ty::Binder<'tcx, T>)
     where
         T: TypeFoldable<'tcx>,
     {
-        struct LateBoundRegionNameCollector<'a>(&'a mut FxHashSet<Symbol>);
-        impl<'tcx> ty::fold::TypeVisitor<'tcx> for LateBoundRegionNameCollector<'_> {
+        debug!("prepare_late_bound_region_info(value: {:?})", value);
+
+        struct LateBoundRegionNameCollector<'a, 'tcx> {
+            used_region_names: &'a mut FxHashSet<Symbol>,
+            type_collector: SsoHashSet<Ty<'tcx>>,
+        }
+
+        impl<'tcx> ty::fold::TypeVisitor<'tcx> for LateBoundRegionNameCollector<'_, 'tcx> {
+            type BreakTy = ();
+
             fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-                if let ty::ReLateBound(_, ty::BoundRegion { kind: ty::BrNamed(_, name) }) = *r {
-                    self.0.insert(name);
+                debug!("LateBoundRegionNameCollector::visit_region(r: {:?}, address: {:p})", r, &r);
+                if let ty::ReLateBound(_, ty::BoundRegion { kind: ty::BrNamed(_, name), .. }) = *r {
+                    self.used_region_names.insert(name);
                 }
                 r.super_visit_with(self)
             }
+
+            // We collect types in order to prevent really large types from compiling for
+            // a really long time. See issue #83150 for why this is necessary.
+            fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
+                debug!("LateBoundRegionNameCollector::visit_ty(ty: {:?}", ty);
+                let not_previously_inserted = self.type_collector.insert(ty);
+                if not_previously_inserted {
+                    ty.super_visit_with(self)
+                } else {
+                    ControlFlow::CONTINUE
+                }
+            }
         }
 
         self.used_region_names.clear();
-        let mut collector = LateBoundRegionNameCollector(&mut self.used_region_names);
+        let mut collector = LateBoundRegionNameCollector {
+            used_region_names: &mut self.used_region_names,
+            type_collector: SsoHashSet::new(),
+        };
         value.visit_with(&mut collector);
         self.region_index = 0;
     }
 }
 
-impl<'tcx, T, P: PrettyPrinter<'tcx>> Print<'tcx, P> for ty::Binder<T>
+impl<'tcx, T, P: PrettyPrinter<'tcx>> Print<'tcx, P> for ty::Binder<'tcx, T>
 where
     T: Print<'tcx, P, Output = P, Error = P::Error> + TypeFoldable<'tcx>,
 {
@@ -1944,28 +2058,28 @@ impl ty::TraitRef<'tcx> {
     }
 }
 
-impl ty::Binder<ty::TraitRef<'tcx>> {
-    pub fn print_only_trait_path(self) -> ty::Binder<TraitRefPrintOnlyTraitPath<'tcx>> {
+impl ty::Binder<'tcx, ty::TraitRef<'tcx>> {
+    pub fn print_only_trait_path(self) -> ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>> {
         self.map_bound(|tr| tr.print_only_trait_path())
     }
 }
 
 forward_display_to_print! {
     Ty<'tcx>,
-    &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+    &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     &'tcx ty::Const<'tcx>,
 
     // HACK(eddyb) these are exhaustive instead of generic,
     // because `for<'tcx>` isn't possible yet.
-    ty::Binder<ty::ExistentialPredicate<'tcx>>,
-    ty::Binder<ty::TraitRef<'tcx>>,
-    ty::Binder<TraitRefPrintOnlyTraitPath<'tcx>>,
-    ty::Binder<ty::FnSig<'tcx>>,
-    ty::Binder<ty::TraitPredicate<'tcx>>,
-    ty::Binder<ty::SubtypePredicate<'tcx>>,
-    ty::Binder<ty::ProjectionPredicate<'tcx>>,
-    ty::Binder<ty::OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>>,
-    ty::Binder<ty::OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>>,
+    ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>,
+    ty::Binder<'tcx, ty::TraitRef<'tcx>>,
+    ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
+    ty::Binder<'tcx, ty::FnSig<'tcx>>,
+    ty::Binder<'tcx, ty::TraitPredicate<'tcx>>,
+    ty::Binder<'tcx, ty::SubtypePredicate<'tcx>>,
+    ty::Binder<'tcx, ty::ProjectionPredicate<'tcx>>,
+    ty::Binder<'tcx, ty::OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>>,
+    ty::Binder<'tcx, ty::OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>>,
 
     ty::OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>,
     ty::OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index 51a214bc07b..c170858ba85 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -14,8 +14,8 @@ use crate::middle::resolve_lifetime::{ObjectLifetimeDefault, Region, ResolveLife
 use crate::middle::stability::{self, DeprecationEntry};
 use crate::mir;
 use crate::mir::interpret::GlobalId;
+use crate::mir::interpret::{ConstAlloc, LitToConstError, LitToConstInput};
 use crate::mir::interpret::{ConstValue, EvalToAllocationRawResult, EvalToConstValueResult};
-use crate::mir::interpret::{LitToConstError, LitToConstInput};
 use crate::mir::mono::CodegenUnit;
 use crate::traits::query::{
     CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
@@ -217,8 +217,11 @@ macro_rules! define_callbacks {
             fn default() -> Self {
                 Providers {
                     $($name: |_, key| bug!(
-                        "`tcx.{}({:?})` unsupported by its crate",
-                         stringify!($name), key
+                        "`tcx.{}({:?})` unsupported by its crate; \
+                         perhaps the `{}` query was never assigned a provider function",
+                        stringify!($name),
+                        key,
+                        stringify!($name),
                     ),)*
                 }
             }
diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
index d0cd8a48f99..416199b3840 100644
--- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
@@ -4,7 +4,6 @@ use crate::mir::{self, interpret};
 use crate::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
 use crate::ty::context::TyCtxt;
 use crate::ty::{self, Ty};
-use rustc_data_structures::fingerprint::{Fingerprint, FingerprintDecoder, FingerprintEncoder};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, OnceCell};
 use rustc_data_structures::thin_vec::ThinVec;
@@ -17,7 +16,7 @@ use rustc_index::vec::{Idx, IndexVec};
 use rustc_query_system::dep_graph::DepContext;
 use rustc_query_system::query::QueryContext;
 use rustc_serialize::{
-    opaque::{self, FileEncodeResult, FileEncoder},
+    opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize},
     Decodable, Decoder, Encodable, Encoder,
 };
 use rustc_session::{CrateDisambiguator, Session};
@@ -526,7 +525,7 @@ impl<'sess> OnDiskCache<'sess> {
     ) {
         let mut current_diagnostics = self.current_diagnostics.borrow_mut();
 
-        let x = current_diagnostics.entry(dep_node_index).or_insert(Vec::new());
+        let x = current_diagnostics.entry(dep_node_index).or_default();
 
         x.extend(Into::<Vec<_>>::into(diagnostics));
     }
@@ -913,12 +912,6 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
     }
 }
 
-impl<'a, 'tcx> FingerprintDecoder for CacheDecoder<'a, 'tcx> {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, Self::Error> {
-        Fingerprint::decode_opaque(&mut self.opaque)
-    }
-}
-
 impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx FxHashSet<LocalDefId> {
     fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Result<Self, String> {
         RefDecodable::decode(d)
@@ -1011,12 +1004,6 @@ where
     }
 }
 
-impl<'a, 'tcx, E: OpaqueEncoder> FingerprintEncoder for CacheEncoder<'a, 'tcx, E> {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> Result<(), E::Error> {
-        self.encoder.encode_fingerprint(f)
-    }
-}
-
 impl<'a, 'tcx, E> Encodable<CacheEncoder<'a, 'tcx, E>> for SyntaxContext
 where
     E: 'a + OpaqueEncoder,
@@ -1045,12 +1032,12 @@ where
     E: 'a + OpaqueEncoder,
 {
     fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> {
-        if *self == DUMMY_SP {
+        let span_data = self.data();
+        if self.is_dummy() {
             TAG_PARTIAL_SPAN.encode(s)?;
-            return SyntaxContext::root().encode(s);
+            return span_data.ctxt.encode(s);
         }
 
-        let span_data = self.data();
         let pos = s.source_map.byte_pos_to_line_and_col(span_data.lo);
         let partial_span = match &pos {
             Some((file_lo, _, _)) => !file_lo.contains(span_data.hi),
@@ -1167,6 +1154,7 @@ where
         emit_f32(f32);
         emit_char(char);
         emit_str(&str);
+        emit_raw_bytes(&[u8]);
     }
 }
 
@@ -1180,42 +1168,6 @@ impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx, FileEncoder>> for [u8] {
     }
 }
 
-// An integer that will always encode to 8 bytes.
-struct IntEncodedWithFixedSize(u64);
-
-impl IntEncodedWithFixedSize {
-    pub const ENCODED_SIZE: usize = 8;
-}
-
-impl<E: OpaqueEncoder> Encodable<E> for IntEncodedWithFixedSize {
-    fn encode(&self, e: &mut E) -> Result<(), E::Error> {
-        let start_pos = e.position();
-        for i in 0..IntEncodedWithFixedSize::ENCODED_SIZE {
-            ((self.0 >> (i * 8)) as u8).encode(e)?;
-        }
-        let end_pos = e.position();
-        assert_eq!((end_pos - start_pos), IntEncodedWithFixedSize::ENCODED_SIZE);
-        Ok(())
-    }
-}
-
-impl<'a> Decodable<opaque::Decoder<'a>> for IntEncodedWithFixedSize {
-    fn decode(decoder: &mut opaque::Decoder<'a>) -> Result<IntEncodedWithFixedSize, String> {
-        let mut value: u64 = 0;
-        let start_pos = decoder.position();
-
-        for i in 0..IntEncodedWithFixedSize::ENCODED_SIZE {
-            let byte: u8 = Decodable::decode(decoder)?;
-            value |= (byte as u64) << (i * 8);
-        }
-
-        let end_pos = decoder.position();
-        assert_eq!((end_pos - start_pos), IntEncodedWithFixedSize::ENCODED_SIZE);
-
-        Ok(IntEncodedWithFixedSize(value))
-    }
-}
-
 pub fn encode_query_results<'a, 'tcx, CTX, Q>(
     tcx: CTX,
     encoder: &mut CacheEncoder<'a, 'tcx, FileEncoder>,
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 315e5d63d2b..b6f93c9bd59 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -4,7 +4,7 @@
 //! types or regions but can be other things. Examples of type relations are
 //! subtyping, type equality, etc.
 
-use crate::mir::interpret::{get_slice_bytes, ConstValue};
+use crate::mir::interpret::{get_slice_bytes, ConstValue, GlobalAlloc, Scalar};
 use crate::ty::error::{ExpectedFound, TypeError};
 use crate::ty::subst::{GenericArg, GenericArgKind, SubstsRef};
 use crate::ty::{self, Ty, TyCtxt, TypeFoldable};
@@ -33,15 +33,6 @@ pub trait TypeRelation<'tcx>: Sized {
     /// relation. Just affects error messages.
     fn a_is_expected(&self) -> bool;
 
-    /// Whether we should look into the substs of unevaluated constants
-    /// even if `feature(const_evaluatable_checked)` is active.
-    ///
-    /// This is needed in `combine` to prevent accidentially creating
-    /// infinite types as we abuse `TypeRelation` to walk a type there.
-    fn visit_ct_substs(&self) -> bool {
-        false
-    }
-
     fn with_cause<F, R>(&mut self, _cause: Cause, f: F) -> R
     where
         F: FnOnce(&mut Self) -> R,
@@ -102,9 +93,9 @@ pub trait TypeRelation<'tcx>: Sized {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>;
 }
@@ -149,7 +140,7 @@ pub fn relate_substs<R: TypeRelation<'tcx>>(
 ) -> RelateResult<'tcx, SubstsRef<'tcx>> {
     let tcx = relation.tcx();
 
-    let params = a_subst.iter().zip(b_subst).enumerate().map(|(i, (a, b))| {
+    let params = iter::zip(a_subst, b_subst).enumerate().map(|(i, (a, b))| {
         let variance = variances.map_or(ty::Invariant, |v| v[i]);
         relation.relate_with_variance(variance, a, b)
     });
@@ -179,12 +170,8 @@ impl<'tcx> Relate<'tcx> for ty::FnSig<'tcx> {
             return Err(TypeError::ArgCount);
         }
 
-        let inputs_and_output = a
-            .inputs()
-            .iter()
-            .cloned()
-            .zip(b.inputs().iter().cloned())
-            .map(|x| (x, false))
+        let inputs_and_output = iter::zip(a.inputs(), b.inputs())
+            .map(|(&a, &b)| ((a, b), false))
             .chain(iter::once(((a.output(), b.output()), true)))
             .map(|((a, b), is_output)| {
                 if is_output {
@@ -192,6 +179,12 @@ impl<'tcx> Relate<'tcx> for ty::FnSig<'tcx> {
                 } else {
                     relation.relate_with_variance(ty::Contravariant, a, b)
                 }
+            })
+            .enumerate()
+            .map(|(i, r)| match r {
+                Err(TypeError::Sorts(exp_found)) => Err(TypeError::ArgumentSorts(exp_found, i)),
+                Err(TypeError::Mutability) => Err(TypeError::ArgumentMutability(i)),
+                r => r,
             });
         Ok(ty::FnSig {
             inputs_and_output: tcx.mk_type_list(inputs_and_output)?,
@@ -308,7 +301,7 @@ impl<'tcx> Relate<'tcx> for GeneratorWitness<'tcx> {
     ) -> RelateResult<'tcx, GeneratorWitness<'tcx>> {
         assert_eq!(a.0.len(), b.0.len());
         let tcx = relation.tcx();
-        let types = tcx.mk_type_list(a.0.iter().zip(b.0).map(|(a, b)| relation.relate(a, b)))?;
+        let types = tcx.mk_type_list(iter::zip(a.0, b.0).map(|(a, b)| relation.relate(a, b)))?;
         Ok(GeneratorWitness(types))
     }
 }
@@ -421,18 +414,20 @@ pub fn super_relate_tys<R: TypeRelation<'tcx>>(
             let t = relation.relate(a_t, b_t)?;
             match relation.relate(sz_a, sz_b) {
                 Ok(sz) => Ok(tcx.mk_ty(ty::Array(t, sz))),
-                // FIXME(#72219) Implement improved diagnostics for mismatched array
-                // length?
-                Err(err) if relation.tcx().lazy_normalization() => Err(err),
                 Err(err) => {
                     // Check whether the lengths are both concrete/known values,
                     // but are unequal, for better diagnostics.
+                    //
+                    // It might seem dubious to eagerly evaluate these constants here,
+                    // we however cannot end up with errors in `Relate` during both
+                    // `type_of` and `predicates_of`. This means that evaluating the
+                    // constants should not cause cycle errors here.
                     let sz_a = sz_a.try_eval_usize(tcx, relation.param_env());
                     let sz_b = sz_b.try_eval_usize(tcx, relation.param_env());
                     match (sz_a, sz_b) {
-                        (Some(sz_a_val), Some(sz_b_val)) => Err(TypeError::FixedArraySize(
-                            expected_found(relation, sz_a_val, sz_b_val),
-                        )),
+                        (Some(sz_a_val), Some(sz_b_val)) if sz_a_val != sz_b_val => Err(
+                            TypeError::FixedArraySize(expected_found(relation, sz_a_val, sz_b_val)),
+                        ),
                         _ => Err(err),
                     }
                 }
@@ -447,7 +442,7 @@ pub fn super_relate_tys<R: TypeRelation<'tcx>>(
         (&ty::Tuple(as_), &ty::Tuple(bs)) => {
             if as_.len() == bs.len() {
                 Ok(tcx.mk_tup(
-                    as_.iter().zip(bs).map(|(a, b)| relation.relate(a.expect_ty(), b.expect_ty())),
+                    iter::zip(as_, bs).map(|(a, b)| relation.relate(a.expect_ty(), b.expect_ty())),
                 )?)
             } else if !(as_.is_empty() || bs.is_empty()) {
                 Err(TypeError::TupleSize(expected_found(relation, as_.len(), bs.len())))
@@ -496,123 +491,116 @@ pub fn super_relate_consts<R: TypeRelation<'tcx>>(
     debug!("{}.super_relate_consts(a = {:?}, b = {:?})", relation.tag(), a, b);
     let tcx = relation.tcx();
 
-    let eagerly_eval = |x: &'tcx ty::Const<'tcx>| x.eval(tcx, relation.param_env()).val;
+    // FIXME(oli-obk): once const generics can have generic types, this assertion
+    // will likely get triggered. Move to `normalize_erasing_regions` at that point.
+    let a_ty = tcx.erase_regions(a.ty);
+    let b_ty = tcx.erase_regions(b.ty);
+    if a_ty != b_ty {
+        relation.tcx().sess.delay_span_bug(
+            DUMMY_SP,
+            &format!("cannot relate constants of different types: {} != {}", a_ty, b_ty),
+        );
+    }
 
-    // FIXME(eddyb) doesn't look like everything below checks that `a.ty == b.ty`.
-    // We could probably always assert it early, as const generic parameters
-    // are not allowed to depend on other generic parameters, i.e. are concrete.
-    // (although there could be normalization differences)
+    let eagerly_eval = |x: &'tcx ty::Const<'tcx>| x.eval(tcx, relation.param_env());
+    let a = eagerly_eval(a);
+    let b = eagerly_eval(b);
 
     // Currently, the values that can be unified are primitive types,
     // and those that derive both `PartialEq` and `Eq`, corresponding
     // to structural-match types.
-    let new_const_val = match (eagerly_eval(a), eagerly_eval(b)) {
+    let is_match = match (a.val, b.val) {
         (ty::ConstKind::Infer(_), _) | (_, ty::ConstKind::Infer(_)) => {
             // The caller should handle these cases!
             bug!("var types encountered in super_relate_consts: {:?} {:?}", a, b)
         }
 
-        (ty::ConstKind::Error(d), _) | (_, ty::ConstKind::Error(d)) => Ok(ty::ConstKind::Error(d)),
+        (ty::ConstKind::Error(_), _) => return Ok(a),
+        (_, ty::ConstKind::Error(_)) => return Ok(b),
 
-        (ty::ConstKind::Param(a_p), ty::ConstKind::Param(b_p)) if a_p.index == b_p.index => {
-            return Ok(a);
+        (ty::ConstKind::Param(a_p), ty::ConstKind::Param(b_p)) => a_p.index == b_p.index,
+        (ty::ConstKind::Placeholder(p1), ty::ConstKind::Placeholder(p2)) => p1 == p2,
+        (ty::ConstKind::Value(a_val), ty::ConstKind::Value(b_val)) => {
+            check_const_value_eq(relation, a_val, b_val, a, b)?
         }
-        (ty::ConstKind::Placeholder(p1), ty::ConstKind::Placeholder(p2)) if p1 == p2 => {
-            return Ok(a);
+
+        (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu))
+            if tcx.features().const_evaluatable_checked =>
+        {
+            tcx.try_unify_abstract_consts(((au.def, au.substs), (bu.def, bu.substs)))
         }
-        (ty::ConstKind::Value(a_val), ty::ConstKind::Value(b_val)) => {
-            let new_val = match (a_val, b_val) {
-                (ConstValue::Scalar(a_val), ConstValue::Scalar(b_val)) if a.ty == b.ty => {
-                    if a_val == b_val {
-                        Ok(ConstValue::Scalar(a_val))
-                    } else if let ty::FnPtr(_) = a.ty.kind() {
-                        let a_instance = tcx.global_alloc(a_val.assert_ptr().alloc_id).unwrap_fn();
-                        let b_instance = tcx.global_alloc(b_val.assert_ptr().alloc_id).unwrap_fn();
-                        if a_instance == b_instance {
-                            Ok(ConstValue::Scalar(a_val))
-                        } else {
-                            Err(TypeError::ConstMismatch(expected_found(relation, a, b)))
-                        }
-                    } else {
-                        Err(TypeError::ConstMismatch(expected_found(relation, a, b)))
-                    }
-                }
 
-                (ConstValue::Slice { .. }, ConstValue::Slice { .. }) => {
-                    let a_bytes = get_slice_bytes(&tcx, a_val);
-                    let b_bytes = get_slice_bytes(&tcx, b_val);
-                    if a_bytes == b_bytes {
-                        Ok(a_val)
-                    } else {
-                        Err(TypeError::ConstMismatch(expected_found(relation, a, b)))
-                    }
-                }
+        // While this is slightly incorrect, it shouldn't matter for `min_const_generics`
+        // and is the better alternative to waiting until `const_evaluatable_checked` can
+        // be stabilized.
+        (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu))
+            if au.def == bu.def && au.promoted == bu.promoted =>
+        {
+            let substs =
+                relation.relate_with_variance(ty::Variance::Invariant, au.substs, bu.substs)?;
+            return Ok(tcx.mk_const(ty::Const {
+                val: ty::ConstKind::Unevaluated(ty::Unevaluated {
+                    def: au.def,
+                    substs,
+                    promoted: au.promoted,
+                }),
+                ty: a.ty,
+            }));
+        }
+        _ => false,
+    };
+    if is_match { Ok(a) } else { Err(TypeError::ConstMismatch(expected_found(relation, a, b))) }
+}
 
-                (ConstValue::ByRef { .. }, ConstValue::ByRef { .. }) => {
-                    match a.ty.kind() {
-                        ty::Array(..) | ty::Adt(..) | ty::Tuple(..) => {
-                            let a_destructured = tcx.destructure_const(relation.param_env().and(a));
-                            let b_destructured = tcx.destructure_const(relation.param_env().and(b));
-
-                            // Both the variant and each field have to be equal.
-                            if a_destructured.variant == b_destructured.variant {
-                                for (a_field, b_field) in
-                                    a_destructured.fields.iter().zip(b_destructured.fields.iter())
-                                {
-                                    relation.consts(a_field, b_field)?;
-                                }
-
-                                Ok(a_val)
-                            } else {
-                                Err(TypeError::ConstMismatch(expected_found(relation, a, b)))
-                            }
-                        }
-                        // FIXME(const_generics): There are probably some `TyKind`s
-                        // which should be handled here.
-                        _ => {
-                            tcx.sess.delay_span_bug(
-                                DUMMY_SP,
-                                &format!("unexpected consts: a: {:?}, b: {:?}", a, b),
-                            );
-                            Err(TypeError::ConstMismatch(expected_found(relation, a, b)))
-                        }
+fn check_const_value_eq<R: TypeRelation<'tcx>>(
+    relation: &mut R,
+    a_val: ConstValue<'tcx>,
+    b_val: ConstValue<'tcx>,
+    // FIXME(oli-obk): these arguments should go away with valtrees
+    a: &'tcx ty::Const<'tcx>,
+    b: &'tcx ty::Const<'tcx>,
+    // FIXME(oli-obk): this should just be `bool` with valtrees
+) -> RelateResult<'tcx, bool> {
+    let tcx = relation.tcx();
+    Ok(match (a_val, b_val) {
+        (ConstValue::Scalar(Scalar::Int(a_val)), ConstValue::Scalar(Scalar::Int(b_val))) => {
+            a_val == b_val
+        }
+        (ConstValue::Scalar(Scalar::Ptr(a_val)), ConstValue::Scalar(Scalar::Ptr(b_val))) => {
+            a_val == b_val
+                || match (tcx.global_alloc(a_val.alloc_id), tcx.global_alloc(b_val.alloc_id)) {
+                    (GlobalAlloc::Function(a_instance), GlobalAlloc::Function(b_instance)) => {
+                        a_instance == b_instance
                     }
+                    _ => false,
                 }
+        }
 
-                _ => Err(TypeError::ConstMismatch(expected_found(relation, a, b))),
-            };
-
-            new_val.map(ty::ConstKind::Value)
+        (ConstValue::Slice { .. }, ConstValue::Slice { .. }) => {
+            get_slice_bytes(&tcx, a_val) == get_slice_bytes(&tcx, b_val)
         }
 
-        (
-            ty::ConstKind::Unevaluated(a_def, a_substs, None),
-            ty::ConstKind::Unevaluated(b_def, b_substs, None),
-        ) if tcx.features().const_evaluatable_checked && !relation.visit_ct_substs() => {
-            if tcx.try_unify_abstract_consts(((a_def, a_substs), (b_def, b_substs))) {
-                Ok(a.val)
+        (ConstValue::ByRef { .. }, ConstValue::ByRef { .. }) => {
+            let a_destructured = tcx.destructure_const(relation.param_env().and(a));
+            let b_destructured = tcx.destructure_const(relation.param_env().and(b));
+
+            // Both the variant and each field have to be equal.
+            if a_destructured.variant == b_destructured.variant {
+                for (a_field, b_field) in iter::zip(a_destructured.fields, b_destructured.fields) {
+                    relation.consts(a_field, b_field)?;
+                }
+
+                true
             } else {
-                Err(TypeError::ConstMismatch(expected_found(relation, a, b)))
+                false
             }
         }
 
-        // While this is slightly incorrect, it shouldn't matter for `min_const_generics`
-        // and is the better alternative to waiting until `const_evaluatable_checked` can
-        // be stabilized.
-        (
-            ty::ConstKind::Unevaluated(a_def, a_substs, a_promoted),
-            ty::ConstKind::Unevaluated(b_def, b_substs, b_promoted),
-        ) if a_def == b_def && a_promoted == b_promoted => {
-            let substs =
-                relation.relate_with_variance(ty::Variance::Invariant, a_substs, b_substs)?;
-            Ok(ty::ConstKind::Unevaluated(a_def, substs, a_promoted))
-        }
-        _ => Err(TypeError::ConstMismatch(expected_found(relation, a, b))),
-    };
-    new_const_val.map(|val| tcx.mk_const(ty::Const { val, ty: a.ty }))
+        _ => false,
+    })
 }
 
-impl<'tcx> Relate<'tcx> for &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>> {
+impl<'tcx> Relate<'tcx> for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>> {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
         a: Self,
@@ -634,13 +622,12 @@ impl<'tcx> Relate<'tcx> for &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'
             return Err(TypeError::ExistentialMismatch(expected_found(relation, a, b)));
         }
 
-        let v = a_v.into_iter().zip(b_v.into_iter()).map(|(ep_a, ep_b)| {
+        let v = iter::zip(a_v, b_v).map(|(ep_a, ep_b)| {
             use crate::ty::ExistentialPredicate::*;
             match (ep_a.skip_binder(), ep_b.skip_binder()) {
-                (Trait(a), Trait(b)) => Ok(ty::Binder::bind(Trait(
-                    relation.relate(ep_a.rebind(a), ep_b.rebind(b))?.skip_binder(),
-                ))),
-                (Projection(a), Projection(b)) => Ok(ty::Binder::bind(Projection(
+                (Trait(a), Trait(b)) => Ok(ep_a
+                    .rebind(Trait(relation.relate(ep_a.rebind(a), ep_b.rebind(b))?.skip_binder()))),
+                (Projection(a), Projection(b)) => Ok(ep_a.rebind(Projection(
                     relation.relate(ep_a.rebind(a), ep_b.rebind(b))?.skip_binder(),
                 ))),
                 (AutoTrait(a), AutoTrait(b)) if a == b => Ok(ep_a.rebind(AutoTrait(a))),
@@ -703,12 +690,12 @@ impl<'tcx> Relate<'tcx> for &'tcx ty::Const<'tcx> {
     }
 }
 
-impl<'tcx, T: Relate<'tcx>> Relate<'tcx> for ty::Binder<T> {
+impl<'tcx, T: Relate<'tcx>> Relate<'tcx> for ty::Binder<'tcx, T> {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>> {
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>> {
         relation.binders(a, b)
     }
 }
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 0ca94a9f180..7290c41d615 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -454,10 +454,16 @@ impl<'a, 'tcx> Lift<'tcx> for ty::PredicateKind<'a> {
     }
 }
 
-impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::Binder<T> {
-    type Lifted = ty::Binder<T::Lifted>;
+impl<'a, 'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::Binder<'a, T>
+where
+    <T as Lift<'tcx>>::Lifted: TypeFoldable<'tcx>,
+{
+    type Lifted = ty::Binder<'tcx, T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        self.map_bound(|v| tcx.lift(v)).transpose()
+        let bound_vars = tcx.lift(self.bound_vars());
+        tcx.lift(self.skip_binder())
+            .zip(bound_vars)
+            .map(|(value, vars)| ty::Binder::bind_with_vars(value, vars))
     }
 }
 
@@ -581,6 +587,7 @@ impl<'a, 'tcx> Lift<'tcx> for ty::error::TypeError<'a> {
             UnsafetyMismatch(x) => UnsafetyMismatch(x),
             AbiMismatch(x) => AbiMismatch(x),
             Mutability => Mutability,
+            ArgumentMutability(i) => ArgumentMutability(i),
             TupleSize(x) => TupleSize(x),
             FixedArraySize(x) => FixedArraySize(x),
             ArgCount => ArgCount,
@@ -601,6 +608,7 @@ impl<'a, 'tcx> Lift<'tcx> for ty::error::TypeError<'a> {
             CyclicTy(t) => return tcx.lift(t).map(|t| CyclicTy(t)),
             CyclicConst(ct) => return tcx.lift(ct).map(|ct| CyclicConst(ct)),
             ProjectionMismatched(x) => ProjectionMismatched(x),
+            ArgumentSorts(x, i) => return tcx.lift(x).map(|x| ArgumentSorts(x, i)),
             Sorts(x) => return tcx.lift(x).map(Sorts),
             ExistentialMismatch(x) => return tcx.lift(x).map(ExistentialMismatch),
             ConstMismatch(x) => return tcx.lift(x).map(ConstMismatch),
@@ -749,7 +757,7 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Box<[T]> {
     }
 }
 
-impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for ty::Binder<T> {
+impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for ty::Binder<'tcx, T> {
     fn super_fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self {
         self.map_bound(|ty| ty.fold_with(folder))
     }
@@ -767,7 +775,7 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for ty::Binder<T> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>> {
+impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>> {
     fn super_fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self {
         ty::util::fold_list(self, folder, |tcx, v| tcx.intern_poly_existential_predicates(v))
     }
@@ -1031,8 +1039,12 @@ impl<'tcx> TypeFoldable<'tcx> for ty::ConstKind<'tcx> {
         match self {
             ty::ConstKind::Infer(ic) => ty::ConstKind::Infer(ic.fold_with(folder)),
             ty::ConstKind::Param(p) => ty::ConstKind::Param(p.fold_with(folder)),
-            ty::ConstKind::Unevaluated(did, substs, promoted) => {
-                ty::ConstKind::Unevaluated(did, substs.fold_with(folder), promoted)
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) => {
+                ty::ConstKind::Unevaluated(ty::Unevaluated {
+                    def,
+                    substs: substs.fold_with(folder),
+                    promoted,
+                })
             }
             ty::ConstKind::Value(_)
             | ty::ConstKind::Bound(..)
@@ -1045,7 +1057,7 @@ impl<'tcx> TypeFoldable<'tcx> for ty::ConstKind<'tcx> {
         match *self {
             ty::ConstKind::Infer(ic) => ic.visit_with(visitor),
             ty::ConstKind::Param(p) => p.visit_with(visitor),
-            ty::ConstKind::Unevaluated(_, substs, _) => substs.visit_with(visitor),
+            ty::ConstKind::Unevaluated(ct) => ct.substs.visit_with(visitor),
             ty::ConstKind::Value(_)
             | ty::ConstKind::Bound(..)
             | ty::ConstKind::Placeholder(_)
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 6c074d3af5c..691bfcc98d1 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -5,6 +5,8 @@
 use self::TyKind::*;
 
 use crate::infer::canonical::Canonical;
+use crate::ty::fold::BoundVarsCollector;
+use crate::ty::fold::ValidateBoundVars;
 use crate::ty::subst::{GenericArg, InternalSubsts, Subst, SubstsRef};
 use crate::ty::InferTy::{self, *};
 use crate::ty::{
@@ -62,22 +64,10 @@ pub enum BoundRegionKind {
 #[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, TyDecodable, Debug, PartialOrd, Ord)]
 #[derive(HashStable)]
 pub struct BoundRegion {
+    pub var: BoundVar,
     pub kind: BoundRegionKind,
 }
 
-impl BoundRegion {
-    /// When canonicalizing, we replace unbound inference variables and free
-    /// regions with anonymous late bound regions. This method asserts that
-    /// we have an anonymous late bound region, which hence may refer to
-    /// a canonical variable.
-    pub fn assert_bound_var(&self) -> BoundVar {
-        match self.kind {
-            BoundRegionKind::BrAnon(var) => BoundVar::from_u32(var),
-            _ => bug!("bound region is not anonymous"),
-        }
-    }
-}
-
 impl BoundRegionKind {
     pub fn is_named(&self) -> bool {
         match *self {
@@ -90,7 +80,7 @@ impl BoundRegionKind {
 /// Defines the kinds of types.
 ///
 /// N.B., if you change this, you'll probably want to change the corresponding
-/// AST structure in `librustc_ast/ast.rs` as well.
+/// AST structure in `rustc_ast/src/ast.rs` as well.
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable, Debug)]
 #[derive(HashStable)]
 #[rustc_diagnostic_item = "TyKind"]
@@ -160,8 +150,8 @@ pub enum TyKind<'tcx> {
     /// ```
     FnPtr(PolyFnSig<'tcx>),
 
-    /// A trait, defined with `trait`.
-    Dynamic(&'tcx List<Binder<ExistentialPredicate<'tcx>>>, ty::Region<'tcx>),
+    /// A trait object. Written as `dyn for<'b> Trait<'b, Assoc = u32> + Send + 'a`.
+    Dynamic(&'tcx List<Binder<'tcx, ExistentialPredicate<'tcx>>>, ty::Region<'tcx>),
 
     /// The anonymous type of a closure. Used to represent the type of
     /// `|a| a`.
@@ -173,7 +163,7 @@ pub enum TyKind<'tcx> {
 
     /// A type representing the types stored inside a generator.
     /// This should only appear in GeneratorInteriors.
-    GeneratorWitness(Binder<&'tcx List<Ty<'tcx>>>),
+    GeneratorWitness(Binder<'tcx, &'tcx List<Ty<'tcx>>>),
 
     /// The never type `!`.
     Never,
@@ -231,8 +221,8 @@ impl TyKind<'tcx> {
 }
 
 // `TyKind` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
-static_assert_size!(TyKind<'_>, 24);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+static_assert_size!(TyKind<'_>, 32);
 
 /// A closure can be modeled as a struct that looks like:
 ///
@@ -747,7 +737,7 @@ impl<'tcx> ExistentialPredicate<'tcx> {
     }
 }
 
-impl<'tcx> Binder<ExistentialPredicate<'tcx>> {
+impl<'tcx> Binder<'tcx, ExistentialPredicate<'tcx>> {
     pub fn with_self_ty(&self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> ty::Predicate<'tcx> {
         use crate::ty::ToPredicate;
         match self.skip_binder() {
@@ -768,7 +758,7 @@ impl<'tcx> Binder<ExistentialPredicate<'tcx>> {
     }
 }
 
-impl<'tcx> List<ty::Binder<ExistentialPredicate<'tcx>>> {
+impl<'tcx> List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>> {
     /// Returns the "principal `DefId`" of this set of existential predicates.
     ///
     /// A Rust trait object type consists (in addition to a lifetime bound)
@@ -794,7 +784,7 @@ impl<'tcx> List<ty::Binder<ExistentialPredicate<'tcx>>> {
     /// is `{Send, Sync}`, while there is no principal. These trait objects
     /// have a "trivial" vtable consisting of just the size, alignment,
     /// and destructor.
-    pub fn principal(&self) -> Option<ty::Binder<ExistentialTraitRef<'tcx>>> {
+    pub fn principal(&self) -> Option<ty::Binder<'tcx, ExistentialTraitRef<'tcx>>> {
         self[0]
             .map_bound(|this| match this {
                 ExistentialPredicate::Trait(tr) => Some(tr),
@@ -810,7 +800,7 @@ impl<'tcx> List<ty::Binder<ExistentialPredicate<'tcx>>> {
     #[inline]
     pub fn projection_bounds<'a>(
         &'a self,
-    ) -> impl Iterator<Item = ty::Binder<ExistentialProjection<'tcx>>> + 'a {
+    ) -> impl Iterator<Item = ty::Binder<'tcx, ExistentialProjection<'tcx>>> + 'a {
         self.iter().filter_map(|predicate| {
             predicate
                 .map_bound(|pred| match pred {
@@ -875,10 +865,10 @@ impl<'tcx> TraitRef<'tcx> {
     }
 }
 
-pub type PolyTraitRef<'tcx> = Binder<TraitRef<'tcx>>;
+pub type PolyTraitRef<'tcx> = Binder<'tcx, TraitRef<'tcx>>;
 
 impl<'tcx> PolyTraitRef<'tcx> {
-    pub fn self_ty(&self) -> Binder<Ty<'tcx>> {
+    pub fn self_ty(&self) -> Binder<'tcx, Ty<'tcx>> {
         self.map_bound_ref(|tr| tr.self_ty())
     }
 
@@ -931,7 +921,7 @@ impl<'tcx> ExistentialTraitRef<'tcx> {
     }
 }
 
-pub type PolyExistentialTraitRef<'tcx> = Binder<ExistentialTraitRef<'tcx>>;
+pub type PolyExistentialTraitRef<'tcx> = Binder<'tcx, ExistentialTraitRef<'tcx>>;
 
 impl<'tcx> PolyExistentialTraitRef<'tcx> {
     pub fn def_id(&self) -> DefId {
@@ -947,52 +937,56 @@ impl<'tcx> PolyExistentialTraitRef<'tcx> {
     }
 }
 
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
+#[derive(HashStable)]
+pub enum BoundVariableKind {
+    Ty(BoundTyKind),
+    Region(BoundRegionKind),
+    Const,
+}
+
 /// Binder is a binder for higher-ranked lifetimes or types. It is part of the
 /// compiler's representation for things like `for<'a> Fn(&'a isize)`
 /// (which would be represented by the type `PolyTraitRef ==
-/// Binder<TraitRef>`). Note that when we instantiate,
+/// Binder<'tcx, TraitRef>`). Note that when we instantiate,
 /// erase, or otherwise "discharge" these bound vars, we change the
-/// type from `Binder<T>` to just `T` (see
+/// type from `Binder<'tcx, T>` to just `T` (see
 /// e.g., `liberate_late_bound_regions`).
 ///
 /// `Decodable` and `Encodable` are implemented for `Binder<T>` using the `impl_binder_encode_decode!` macro.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-pub struct Binder<T>(T);
+pub struct Binder<'tcx, T>(T, &'tcx List<BoundVariableKind>);
 
-impl<T> Binder<T> {
+impl<'tcx, T> Binder<'tcx, T>
+where
+    T: TypeFoldable<'tcx>,
+{
     /// Wraps `value` in a binder, asserting that `value` does not
     /// contain any bound vars that would be bound by the
     /// binder. This is commonly used to 'inject' a value T into a
     /// different binding level.
-    pub fn dummy<'tcx>(value: T) -> Binder<T>
-    where
-        T: TypeFoldable<'tcx>,
-    {
+    pub fn dummy(value: T) -> Binder<'tcx, T> {
         debug_assert!(!value.has_escaping_bound_vars());
-        Binder(value)
+        Binder(value, ty::List::empty())
     }
 
     /// Wraps `value` in a binder, binding higher-ranked vars (if any).
-    pub fn bind(value: T) -> Binder<T> {
-        Binder(value)
+    pub fn bind(value: T, tcx: TyCtxt<'tcx>) -> Binder<'tcx, T> {
+        let mut collector = BoundVarsCollector::new();
+        value.visit_with(&mut collector);
+        Binder(value, collector.into_vars(tcx))
     }
 
-    /// Wraps `value` in a binder without actually binding any currently
-    /// unbound variables.
-    ///
-    /// Note that this will shift all debrujin indices of escaping bound variables
-    /// by 1 to avoid accidential captures.
-    pub fn wrap_nonbinding(tcx: TyCtxt<'tcx>, value: T) -> Binder<T>
-    where
-        T: TypeFoldable<'tcx>,
-    {
-        if value.has_escaping_bound_vars() {
-            Binder::bind(super::fold::shift_vars(tcx, value, 1))
-        } else {
-            Binder::dummy(value)
+    pub fn bind_with_vars(value: T, vars: &'tcx List<BoundVariableKind>) -> Binder<'tcx, T> {
+        if cfg!(debug_assertions) {
+            let mut validator = ValidateBoundVars::new(vars);
+            value.visit_with(&mut validator);
         }
+        Binder(value, vars)
     }
+}
 
+impl<'tcx, T> Binder<'tcx, T> {
     /// Skips the binder and returns the "bound" value. This is a
     /// risky thing to do because it's easy to get confused about
     /// De Bruijn indices and the like. It is usually better to
@@ -1013,22 +1007,39 @@ impl<T> Binder<T> {
         self.0
     }
 
-    pub fn as_ref(&self) -> Binder<&T> {
-        Binder(&self.0)
+    pub fn bound_vars(&self) -> &'tcx List<BoundVariableKind> {
+        self.1
     }
 
-    pub fn map_bound_ref<F, U>(&self, f: F) -> Binder<U>
+    pub fn as_ref(&self) -> Binder<'tcx, &T> {
+        Binder(&self.0, self.1)
+    }
+
+    pub fn map_bound_ref_unchecked<F, U>(&self, f: F) -> Binder<'tcx, U>
+    where
+        F: FnOnce(&T) -> U,
+    {
+        let value = f(&self.0);
+        Binder(value, self.1)
+    }
+
+    pub fn map_bound_ref<F, U: TypeFoldable<'tcx>>(&self, f: F) -> Binder<'tcx, U>
     where
         F: FnOnce(&T) -> U,
     {
         self.as_ref().map_bound(f)
     }
 
-    pub fn map_bound<F, U>(self, f: F) -> Binder<U>
+    pub fn map_bound<F, U: TypeFoldable<'tcx>>(self, f: F) -> Binder<'tcx, U>
     where
         F: FnOnce(T) -> U,
     {
-        Binder(f(self.0))
+        let value = f(self.0);
+        if cfg!(debug_assertions) {
+            let mut validator = ValidateBoundVars::new(self.1);
+            value.visit_with(&mut validator);
+        }
+        Binder(value, self.1)
     }
 
     /// Wraps a `value` in a binder, using the same bound variables as the
@@ -1040,8 +1051,15 @@ impl<T> Binder<T> {
     /// don't actually track bound vars. However, semantically, it is different
     /// because bound vars aren't allowed to change here, whereas they are
     /// in `bind`. This may be (debug) asserted in the future.
-    pub fn rebind<U>(&self, value: U) -> Binder<U> {
-        Binder(value)
+    pub fn rebind<U>(&self, value: U) -> Binder<'tcx, U>
+    where
+        U: TypeFoldable<'tcx>,
+    {
+        if cfg!(debug_assertions) {
+            let mut validator = ValidateBoundVars::new(self.bound_vars());
+            value.visit_with(&mut validator);
+        }
+        Binder(value, self.1)
     }
 
     /// Unwraps and returns the value within, but only if it contains
@@ -1054,45 +1072,32 @@ impl<T> Binder<T> {
     /// binders, but that would require adjusting the debruijn
     /// indices, and given the shallow binding structure we often use,
     /// would not be that useful.)
-    pub fn no_bound_vars<'tcx>(self) -> Option<T>
+    pub fn no_bound_vars(self) -> Option<T>
     where
         T: TypeFoldable<'tcx>,
     {
         if self.0.has_escaping_bound_vars() { None } else { Some(self.skip_binder()) }
     }
 
-    /// Given two things that have the same binder level,
-    /// and an operation that wraps on their contents, executes the operation
-    /// and then wraps its result.
-    ///
-    /// `f` should consider bound regions at depth 1 to be free, and
-    /// anything it produces with bound regions at depth 1 will be
-    /// bound in the resulting return value.
-    pub fn fuse<U, F, R>(self, u: Binder<U>, f: F) -> Binder<R>
-    where
-        F: FnOnce(T, U) -> R,
-    {
-        Binder(f(self.0, u.0))
-    }
-
     /// Splits the contents into two things that share the same binder
     /// level as the original, returning two distinct binders.
     ///
     /// `f` should consider bound regions at depth 1 to be free, and
     /// anything it produces with bound regions at depth 1 will be
     /// bound in the resulting return values.
-    pub fn split<U, V, F>(self, f: F) -> (Binder<U>, Binder<V>)
+    pub fn split<U, V, F>(self, f: F) -> (Binder<'tcx, U>, Binder<'tcx, V>)
     where
         F: FnOnce(T) -> (U, V),
     {
         let (u, v) = f(self.0);
-        (Binder(u), Binder(v))
+        (Binder(u, self.1), Binder(v, self.1))
     }
 }
 
-impl<T> Binder<Option<T>> {
-    pub fn transpose(self) -> Option<Binder<T>> {
-        self.0.map(Binder)
+impl<'tcx, T> Binder<'tcx, Option<T>> {
+    pub fn transpose(self) -> Option<Binder<'tcx, T>> {
+        let bound_vars = self.1;
+        self.0.map(|v| Binder(v, bound_vars))
     }
 }
 
@@ -1155,19 +1160,7 @@ pub struct GenSig<'tcx> {
     pub return_ty: Ty<'tcx>,
 }
 
-pub type PolyGenSig<'tcx> = Binder<GenSig<'tcx>>;
-
-impl<'tcx> PolyGenSig<'tcx> {
-    pub fn resume_ty(&self) -> ty::Binder<Ty<'tcx>> {
-        self.map_bound_ref(|sig| sig.resume_ty)
-    }
-    pub fn yield_ty(&self) -> ty::Binder<Ty<'tcx>> {
-        self.map_bound_ref(|sig| sig.yield_ty)
-    }
-    pub fn return_ty(&self) -> ty::Binder<Ty<'tcx>> {
-        self.map_bound_ref(|sig| sig.return_ty)
-    }
-}
+pub type PolyGenSig<'tcx> = Binder<'tcx, GenSig<'tcx>>;
 
 /// Signature of a function type, which we have arbitrarily
 /// decided to use to refer to the input/output types.
@@ -1205,22 +1198,22 @@ impl<'tcx> FnSig<'tcx> {
     }
 }
 
-pub type PolyFnSig<'tcx> = Binder<FnSig<'tcx>>;
+pub type PolyFnSig<'tcx> = Binder<'tcx, FnSig<'tcx>>;
 
 impl<'tcx> PolyFnSig<'tcx> {
     #[inline]
-    pub fn inputs(&self) -> Binder<&'tcx [Ty<'tcx>]> {
-        self.map_bound_ref(|fn_sig| fn_sig.inputs())
+    pub fn inputs(&self) -> Binder<'tcx, &'tcx [Ty<'tcx>]> {
+        self.map_bound_ref_unchecked(|fn_sig| fn_sig.inputs())
     }
     #[inline]
-    pub fn input(&self, index: usize) -> ty::Binder<Ty<'tcx>> {
+    pub fn input(&self, index: usize) -> ty::Binder<'tcx, Ty<'tcx>> {
         self.map_bound_ref(|fn_sig| fn_sig.inputs()[index])
     }
-    pub fn inputs_and_output(&self) -> ty::Binder<&'tcx List<Ty<'tcx>>> {
+    pub fn inputs_and_output(&self) -> ty::Binder<'tcx, &'tcx List<Ty<'tcx>>> {
         self.map_bound_ref(|fn_sig| fn_sig.inputs_and_output)
     }
     #[inline]
-    pub fn output(&self) -> ty::Binder<Ty<'tcx>> {
+    pub fn output(&self) -> ty::Binder<'tcx, Ty<'tcx>> {
         self.map_bound_ref(|fn_sig| fn_sig.output())
     }
     pub fn c_variadic(&self) -> bool {
@@ -1234,7 +1227,7 @@ impl<'tcx> PolyFnSig<'tcx> {
     }
 }
 
-pub type CanonicalPolyFnSig<'tcx> = Canonical<'tcx, Binder<FnSig<'tcx>>>;
+pub type CanonicalPolyFnSig<'tcx> = Canonical<'tcx, Binder<'tcx, FnSig<'tcx>>>;
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
@@ -1248,10 +1241,6 @@ impl<'tcx> ParamTy {
         ParamTy { index, name }
     }
 
-    pub fn for_self() -> ParamTy {
-        ParamTy::new(0, kw::SelfUpper)
-    }
-
     pub fn for_def(def: &ty::GenericParamDef) -> ParamTy {
         ParamTy::new(def.index, def.name)
     }
@@ -1269,7 +1258,7 @@ pub struct ParamConst {
     pub name: Symbol,
 }
 
-impl<'tcx> ParamConst {
+impl ParamConst {
     pub fn new(index: u32, name: Symbol) -> ParamConst {
         ParamConst { index, name }
     }
@@ -1277,10 +1266,6 @@ impl<'tcx> ParamConst {
     pub fn for_def(def: &ty::GenericParamDef) -> ParamConst {
         ParamConst::new(def.index, def.name)
     }
-
-    pub fn to_const(self, tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> &'tcx ty::Const<'tcx> {
-        tcx.mk_const_param(self.index, self.name, ty)
-    }
 }
 
 pub type Region<'tcx> = &'tcx RegionKind;
@@ -1486,7 +1471,7 @@ pub struct ExistentialProjection<'tcx> {
     pub ty: Ty<'tcx>,
 }
 
-pub type PolyExistentialProjection<'tcx> = Binder<ExistentialProjection<'tcx>>;
+pub type PolyExistentialProjection<'tcx> = Binder<'tcx, ExistentialProjection<'tcx>>;
 
 impl<'tcx> ExistentialProjection<'tcx> {
     /// Extracts the underlying existential trait reference from this projection.
@@ -1580,35 +1565,6 @@ impl RegionKind {
         }
     }
 
-    /// Adjusts any De Bruijn indices so as to make `to_binder` the
-    /// innermost binder. That is, if we have something bound at `to_binder`,
-    /// it will now be bound at INNERMOST. This is an appropriate thing to do
-    /// when moving a region out from inside binders:
-    ///
-    /// ```
-    ///             for<'a>   fn(for<'b>   for<'c>   fn(&'a u32), _)
-    /// // Binder:  D3           D2        D1            ^^
-    /// ```
-    ///
-    /// Here, the region `'a` would have the De Bruijn index D3,
-    /// because it is the bound 3 binders out. However, if we wanted
-    /// to refer to that region `'a` in the second argument (the `_`),
-    /// those two binders would not be in scope. In that case, we
-    /// might invoke `shift_out_to_binder(D3)`. This would adjust the
-    /// De Bruijn index of `'a` to D1 (the innermost binder).
-    ///
-    /// If we invoke `shift_out_to_binder` and the region is in fact
-    /// bound by one of the binders we are shifting out of, that is an
-    /// error (and should fail an assertion failure).
-    pub fn shifted_out_to_binder(&self, to_binder: ty::DebruijnIndex) -> RegionKind {
-        match *self {
-            ty::ReLateBound(debruijn, r) => {
-                ty::ReLateBound(debruijn.shifted_out_to_binder(to_binder), r)
-            }
-            r => r,
-        }
-    }
-
     pub fn type_flags(&self) -> TypeFlags {
         let mut flags = TypeFlags::empty();
 
@@ -2160,7 +2116,7 @@ impl<'tcx> TyS<'tcx> {
     ///
     /// Note that during type checking, we use an inference variable
     /// to represent the closure kind, because it has not yet been
-    /// inferred. Once upvar inference (in `src/librustc_typeck/check/upvar.rs`)
+    /// inferred. Once upvar inference (in `rustc_typeck/src/check/upvar.rs`)
     /// is complete, that type variable will be unified.
     pub fn to_opt_closure_kind(&self) -> Option<ty::ClosureKind> {
         match self.kind() {
diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs
index 5d1b976ae97..c84ca61122f 100644
--- a/compiler/rustc_middle/src/ty/subst.rs
+++ b/compiler/rustc_middle/src/ty/subst.rs
@@ -1,5 +1,6 @@
 // Type substitutions.
 
+use crate::mir;
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{TypeFoldable, TypeFolder, TypeVisitor};
 use crate::ty::sty::{ClosureSubsts, GeneratorSubsts};
@@ -448,7 +449,10 @@ impl<'a, 'tcx> TypeFolder<'tcx> for SubstFolder<'a, 'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: ty::Binder<T>) -> ty::Binder<T> {
+    fn fold_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: ty::Binder<'tcx, T>,
+    ) -> ty::Binder<'tcx, T> {
         self.binders_passed += 1;
         let t = t.super_fold_with(self);
         self.binders_passed -= 1;
@@ -503,6 +507,11 @@ impl<'a, 'tcx> TypeFolder<'tcx> for SubstFolder<'a, 'tcx> {
             c.super_fold_with(self)
         }
     }
+
+    #[inline]
+    fn fold_mir_const(&mut self, c: mir::ConstantKind<'tcx>) -> mir::ConstantKind<'tcx> {
+        c.super_fold_with(self)
+    }
 }
 
 impl<'a, 'tcx> SubstFolder<'a, 'tcx> {
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index ce17a724e25..33065bc3a7b 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -69,6 +69,12 @@ pub struct TraitImpls {
     non_blanket_impls: FxHashMap<fast_reject::SimplifiedType, Vec<DefId>>,
 }
 
+impl TraitImpls {
+    pub fn blanket_impls(&self) -> &[DefId] {
+        self.blanket_impls.as_slice()
+    }
+}
+
 impl<'tcx> TraitDef {
     pub fn new(
         def_id: DefId,
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 8edde8794ed..7d09ca5152f 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -21,7 +21,7 @@ use rustc_macros::HashStable;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{Integer, Size, TargetDataLayout};
 use smallvec::SmallVec;
-use std::{cmp, fmt};
+use std::{cmp, fmt, iter};
 
 #[derive(Copy, Clone, Debug)]
 pub struct Discr<'tcx> {
@@ -414,9 +414,7 @@ impl<'tcx> TyCtxt<'tcx> {
             _ => bug!(),
         };
 
-        let result = item_substs
-            .iter()
-            .zip(impl_substs.iter())
+        let result = iter::zip(item_substs, impl_substs)
             .filter(|&(_, k)| {
                 match k.unpack() {
                     GenericArgKind::Lifetime(&ty::RegionKind::ReEarlyBound(ref ebr)) => {
@@ -501,10 +499,9 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         closure_def_id: DefId,
         closure_substs: SubstsRef<'tcx>,
-    ) -> Option<ty::Binder<Ty<'tcx>>> {
+        env_region: ty::RegionKind,
+    ) -> Option<Ty<'tcx>> {
         let closure_ty = self.mk_closure(closure_def_id, closure_substs);
-        let br = ty::BoundRegion { kind: ty::BrEnv };
-        let env_region = ty::ReLateBound(ty::INNERMOST, br);
         let closure_kind_ty = closure_substs.as_closure().kind_ty();
         let closure_kind = closure_kind_ty.to_opt_closure_kind()?;
         let env_ty = match closure_kind {
@@ -512,7 +509,7 @@ impl<'tcx> TyCtxt<'tcx> {
             ty::ClosureKind::FnMut => self.mk_mut_ref(self.mk_region(env_region), closure_ty),
             ty::ClosureKind::FnOnce => closure_ty,
         };
-        Some(ty::Binder::bind(env_ty))
+        Some(env_ty)
     }
 
     /// Returns `true` if the node pointed to by `def_id` is a `static` item.
@@ -701,7 +698,6 @@ impl<'tcx> ty::TyS<'tcx> {
     /// optimization as well as the rules around static values. Note
     /// that the `Freeze` trait is not exposed to end users and is
     /// effectively an implementation detail.
-    // FIXME: use `TyCtxtAt` instead of separate `Span`.
     pub fn is_freeze(&'tcx self, tcx_at: TyCtxtAt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool {
         self.is_trivially_freeze() || tcx_at.is_freeze_raw(param_env.and(self))
     }
@@ -741,6 +737,46 @@ impl<'tcx> ty::TyS<'tcx> {
         }
     }
 
+    /// Checks whether values of this type `T` implement the `Unpin` trait.
+    pub fn is_unpin(&'tcx self, tcx_at: TyCtxtAt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool {
+        self.is_trivially_unpin() || tcx_at.is_unpin_raw(param_env.and(self))
+    }
+
+    /// Fast path helper for testing if a type is `Unpin`.
+    ///
+    /// Returning true means the type is known to be `Unpin`. Returning
+    /// `false` means nothing -- could be `Unpin`, might not be.
+    fn is_trivially_unpin(&self) -> bool {
+        match self.kind() {
+            ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Bool
+            | ty::Char
+            | ty::Str
+            | ty::Never
+            | ty::Ref(..)
+            | ty::RawPtr(_)
+            | ty::FnDef(..)
+            | ty::Error(_)
+            | ty::FnPtr(_) => true,
+            ty::Tuple(_) => self.tuple_fields().all(Self::is_trivially_unpin),
+            ty::Slice(elem_ty) | ty::Array(elem_ty, _) => elem_ty.is_trivially_unpin(),
+            ty::Adt(..)
+            | ty::Bound(..)
+            | ty::Closure(..)
+            | ty::Dynamic(..)
+            | ty::Foreign(_)
+            | ty::Generator(..)
+            | ty::GeneratorWitness(_)
+            | ty::Infer(_)
+            | ty::Opaque(..)
+            | ty::Param(_)
+            | ty::Placeholder(_)
+            | ty::Projection(_) => false,
+        }
+    }
+
     /// If `ty.needs_drop(...)` returns `true`, then `ty` is definitely
     /// non-copy and *might* have a destructor attached; if it returns
     /// `false`, then `ty` definitely has no destructor (i.e., no drop glue).
diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs
index bb7fc661d2d..c2fe5f1ef3f 100644
--- a/compiler/rustc_middle/src/ty/walk.rs
+++ b/compiler/rustc_middle/src/ty/walk.rs
@@ -195,8 +195,8 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
                 | ty::ConstKind::Value(_)
                 | ty::ConstKind::Error(_) => {}
 
-                ty::ConstKind::Unevaluated(_, substs, _) => {
-                    stack.extend(substs.iter().rev());
+                ty::ConstKind::Unevaluated(ct) => {
+                    stack.extend(ct.substs.iter().rev());
                 }
             }
         }
diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs
index b0b58a8d003..9f19a474ca3 100644
--- a/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs
@@ -10,16 +10,18 @@ use rustc_middle::mir::{
     FakeReadCause, Local, LocalDecl, LocalInfo, LocalKind, Location, Operand, Place, PlaceRef,
     ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, VarBindingForm,
 };
-use rustc_middle::ty::{self, suggest_constraining_type_param, Instance, Ty};
-use rustc_span::{source_map::DesugaringKind, symbol::sym, Span};
+use rustc_middle::ty::{self, suggest_constraining_type_param, Ty, TypeFoldable};
+use rustc_span::source_map::DesugaringKind;
+use rustc_span::symbol::sym;
+use rustc_span::{Span, DUMMY_SP};
 
 use crate::dataflow::drop_flag_effects;
 use crate::dataflow::indexes::{MoveOutIndex, MovePathIndex};
 use crate::util::borrowck_errors;
 
 use crate::borrow_check::{
-    borrow_set::BorrowData, prefixes::IsPrefixOf, InitializationRequiringAction, MirBorrowckCtxt,
-    PrefixSet, WriteKind,
+    borrow_set::BorrowData, diagnostics::Instance, prefixes::IsPrefixOf,
+    InitializationRequiringAction, MirBorrowckCtxt, PrefixSet, WriteKind,
 };
 
 use super::{
@@ -195,7 +197,11 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                 );
                             }
                         }
-                        FnSelfUseKind::Normal { self_arg, implicit_into_iter } => {
+                        FnSelfUseKind::Normal {
+                            self_arg,
+                            implicit_into_iter,
+                            is_option_or_result,
+                        } => {
                             if implicit_into_iter {
                                 err.span_label(
                                     fn_call_span,
@@ -213,13 +219,22 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                     ),
                                 );
                             }
+                            if is_option_or_result {
+                                err.span_suggestion_verbose(
+                                    fn_call_span.shrink_to_lo(),
+                                    "consider calling `.as_ref()` to borrow the type's contents",
+                                    "as_ref().".to_string(),
+                                    Applicability::MachineApplicable,
+                                );
+                            }
                             // Avoid pointing to the same function in multiple different
-                            // error messages
-                            if self.fn_self_span_reported.insert(self_arg.span) {
+                            // error messages.
+                            if span != DUMMY_SP && self.fn_self_span_reported.insert(self_arg.span)
+                            {
                                 err.span_note(
-                                    self_arg.span,
-                                    &format!("this function takes ownership of the receiver `self`, which moves {}", place_name)
-                                );
+                                        self_arg.span,
+                                        &format!("this function takes ownership of the receiver `self`, which moves {}", place_name)
+                                    );
                             }
                         }
                         // Deref::deref takes &self, which cannot cause a move
@@ -261,7 +276,24 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
                 if let Some(DesugaringKind::ForLoop(_)) = move_span.desugaring_kind() {
                     let sess = self.infcx.tcx.sess;
-                    if let Ok(snippet) = sess.source_map().span_to_snippet(move_span) {
+                    let ty = used_place.ty(self.body, self.infcx.tcx).ty;
+                    // If we have a `&mut` ref, we need to reborrow.
+                    if let ty::Ref(_, _, hir::Mutability::Mut) = ty.kind() {
+                        // If we are in a loop this will be suggested later.
+                        if !is_loop_move {
+                            err.span_suggestion_verbose(
+                                move_span.shrink_to_lo(),
+                                &format!(
+                                    "consider creating a fresh reborrow of {} here",
+                                    self.describe_place(moved_place.as_ref())
+                                        .map(|n| format!("`{}`", n))
+                                        .unwrap_or_else(|| "the mutable reference".to_string()),
+                                ),
+                                format!("&mut *"),
+                                Applicability::MachineApplicable,
+                            );
+                        }
+                    } else if let Ok(snippet) = sess.source_map().span_to_snippet(move_span) {
                         err.span_suggestion(
                             move_span,
                             "consider borrowing to avoid moving into the for loop",
@@ -1267,6 +1299,29 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         if return_span != borrow_span {
             err.span_label(borrow_span, note);
+
+            let tcx = self.infcx.tcx;
+            let ty_params = ty::List::empty();
+
+            let return_ty = self.regioncx.universal_regions().unnormalized_output_ty;
+            let return_ty = tcx.erase_regions(return_ty);
+
+            // to avoid panics
+            if !return_ty.has_infer_types() {
+                if let Some(iter_trait) = tcx.get_diagnostic_item(sym::Iterator) {
+                    if tcx.type_implements_trait((iter_trait, return_ty, ty_params, self.param_env))
+                    {
+                        if let Ok(snippet) = tcx.sess.source_map().span_to_snippet(return_span) {
+                            err.span_suggestion_hidden(
+                                return_span,
+                                "use `.collect()` to allocate the iterator",
+                                format!("{}{}", snippet, ".collect::<Vec<_>>()"),
+                                Applicability::MaybeIncorrect,
+                            );
+                        }
+                    }
+                }
+            }
         }
 
         Some(err)
@@ -1638,7 +1693,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 if decl.can_be_made_mutable() {
                     err.span_suggestion(
                         decl.source_info.span,
-                        "make this binding mutable",
+                        "consider making this binding mutable",
                         format!("mut {}", name),
                         Applicability::MachineApplicable,
                     );
@@ -1702,7 +1757,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         impl<'tcx> Visitor<'tcx> for FakeReadCauseFinder<'tcx> {
             fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
                 match statement {
-                    Statement { kind: StatementKind::FakeRead(cause, box place), .. }
+                    Statement { kind: StatementKind::FakeRead(box (cause, place)), .. }
                         if *place == self.place =>
                     {
                         self.cause = Some(*cause);
diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs
index 06e3f4b91f6..2a388b8a72b 100644
--- a/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs
@@ -515,7 +515,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 let block = &self.body.basic_blocks()[location.block];
 
                 let kind = if let Some(&Statement {
-                    kind: StatementKind::FakeRead(FakeReadCause::ForLet, _),
+                    kind: StatementKind::FakeRead(box (FakeReadCause::ForLet(_), _)),
                     ..
                 }) = block.statements.get(location.statement_index)
                 {
diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs
index 04ea3cbd8b6..aa9f18d9996 100644
--- a/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs
+++ b/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs
@@ -7,8 +7,8 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItemGroup;
 use rustc_hir::GeneratorKind;
 use rustc_middle::mir::{
-    AggregateKind, Constant, Field, Local, LocalInfo, LocalKind, Location, Operand, Place,
-    PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
+    AggregateKind, Constant, FakeReadCause, Field, Local, LocalInfo, LocalKind, Location, Operand,
+    Place, PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
 };
 use rustc_middle::ty::print::Print;
 use rustc_middle::ty::{self, DefIdTree, Instance, Ty, TyCtxt};
@@ -18,6 +18,7 @@ use rustc_span::{
     Span,
 };
 use rustc_target::abi::VariantIdx;
+use std::iter;
 
 use super::borrow_set::BorrowData;
 use super::MirBorrowckCtxt;
@@ -81,12 +82,12 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let terminator = self.body[location.block].terminator();
         debug!("add_moved_or_invoked_closure_note: terminator={:?}", terminator);
         if let TerminatorKind::Call {
-            func: Operand::Constant(box Constant { literal: ty::Const { ty: const_ty, .. }, .. }),
+            func: Operand::Constant(box Constant { literal, .. }),
             args,
             ..
         } = &terminator.kind
         {
-            if let ty::FnDef(id, _) = *const_ty.kind() {
+            if let ty::FnDef(id, _) = *literal.ty().kind() {
                 debug!("add_moved_or_invoked_closure_note: id={:?}", id);
                 if self.infcx.tcx.parent(id) == self.infcx.tcx.lang_items().fn_once_trait() {
                     let closure = match args.first() {
@@ -388,10 +389,14 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     // so it's safe to call `expect_local`.
                     //
                     // We know the field exists so it's safe to call operator[] and `unwrap` here.
-                    let (&var_id, _) =
-                        self.infcx.tcx.typeck(def_id.expect_local()).closure_captures[&def_id]
-                            .get_index(field.index())
-                            .unwrap();
+                    let var_id = self
+                        .infcx
+                        .tcx
+                        .typeck(def_id.expect_local())
+                        .closure_min_captures_flattened(def_id)
+                        .nth(field.index())
+                        .unwrap()
+                        .get_root_variable();
 
                     self.infcx.tcx.hir().name(var_id).to_string()
                 }
@@ -497,7 +502,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         // lifetimes without names with the value `'0`.
         match ty.kind() {
             ty::Ref(
-                ty::RegionKind::ReLateBound(_, ty::BoundRegion { kind: br })
+                ty::RegionKind::ReLateBound(_, ty::BoundRegion { kind: br, .. })
                 | ty::RegionKind::RePlaceholder(ty::PlaceholderRegion { name: br, .. }),
                 _,
                 _,
@@ -518,7 +523,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let region = match ty.kind() {
             ty::Ref(region, _, _) => {
                 match region {
-                    ty::RegionKind::ReLateBound(_, ty::BoundRegion { kind: br })
+                    ty::RegionKind::ReLateBound(_, ty::BoundRegion { kind: br, .. })
                     | ty::RegionKind::RePlaceholder(ty::PlaceholderRegion { name: br, .. }) => {
                         printer.region_highlight_mode.highlighting_bound_region(*br, counter)
                     }
@@ -568,7 +573,13 @@ pub(super) enum UseSpans<'tcx> {
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub(super) enum FnSelfUseKind<'tcx> {
     /// A normal method call of the form `receiver.foo(a, b, c)`
-    Normal { self_arg: Ident, implicit_into_iter: bool },
+    Normal {
+        self_arg: Ident,
+        implicit_into_iter: bool,
+        /// Whether the self type of the method call has an `.as_ref()` method.
+        /// Used for better diagnostics.
+        is_option_or_result: bool,
+    },
     /// A call to `FnOnce::call_once`, desugared from `my_closure(a, b, c)`
     FnOnceCall,
     /// A call to an operator trait, desuraged from operator syntax (e.g. `a << b`)
@@ -790,6 +801,24 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             }
         }
 
+        // StatementKind::FakeRead only contains a def_id if they are introduced as a result
+        // of pattern matching within a closure.
+        if let StatementKind::FakeRead(box (cause, ref place)) = stmt.kind {
+            match cause {
+                FakeReadCause::ForMatchedPlace(Some(closure_def_id))
+                | FakeReadCause::ForLet(Some(closure_def_id)) => {
+                    debug!("move_spans: def_id={:?} place={:?}", closure_def_id, place);
+                    let places = &[Operand::Move(*place)];
+                    if let Some((args_span, generator_kind, var_span)) =
+                        self.closure_span(closure_def_id, moved_place, places)
+                    {
+                        return ClosureUse { generator_kind, args_span, var_span };
+                    }
+                }
+                _ => {}
+            }
+        }
+
         let normal_ret =
             if moved_place.projection.iter().any(|p| matches!(p, ProjectionElem::Downcast(..))) {
                 PatUse(stmt.source_info.span)
@@ -877,7 +906,17 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     fn_call_span.desugaring_kind(),
                     Some(DesugaringKind::ForLoop(ForLoopLoc::IntoIter))
                 );
-                FnSelfUseKind::Normal { self_arg, implicit_into_iter }
+                let parent_self_ty = parent
+                    .filter(|did| tcx.def_kind(*did) == rustc_hir::def::DefKind::Impl)
+                    .and_then(|did| match tcx.type_of(did).kind() {
+                        ty::Adt(def, ..) => Some(def.did),
+                        _ => None,
+                    });
+                let is_option_or_result = parent_self_ty.map_or(false, |def_id| {
+                    tcx.is_diagnostic_item(sym::option_type, def_id)
+                        || tcx.is_diagnostic_item(sym::result_type, def_id)
+                });
+                FnSelfUseKind::Normal { self_arg, implicit_into_iter, is_option_or_result }
             });
 
             return FnSelfUse {
@@ -966,12 +1005,13 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let expr = &self.infcx.tcx.hir().expect_expr(hir_id).kind;
         debug!("closure_span: hir_id={:?} expr={:?}", hir_id, expr);
         if let hir::ExprKind::Closure(.., body_id, args_span, _) = expr {
-            for (upvar_hir_id, place) in
-                self.infcx.tcx.typeck(def_id.expect_local()).closure_captures[&def_id]
-                    .keys()
-                    .zip(places)
-            {
-                let span = self.infcx.tcx.upvars_mentioned(local_did)?[upvar_hir_id].span;
+            for (captured_place, place) in iter::zip(
+                self.infcx.tcx.typeck(def_id.expect_local()).closure_min_captures_flattened(def_id),
+                places,
+            ) {
+                let upvar_hir_id = captured_place.get_root_variable();
+                //FIXME(project-rfc-2229#8): Use better span from captured_place
+                let span = self.infcx.tcx.upvars_mentioned(local_did)?[&upvar_hir_id].span;
                 match place {
                     Operand::Copy(place) | Operand::Move(place)
                         if target_place == place.as_ref() =>
@@ -979,10 +1019,6 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         debug!("closure_span: found captured local {:?}", place);
                         let body = self.infcx.tcx.hir().body(*body_id);
                         let generator_kind = body.generator_kind();
-                        let upvar_id = ty::UpvarId {
-                            var_path: ty::UpvarPath { hir_id: *upvar_hir_id },
-                            closure_expr_id: local_did,
-                        };
 
                         // If we have a more specific span available, point to that.
                         // We do this even though this span might be part of a borrow error
@@ -990,11 +1026,11 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         // to a span that shows why the upvar is used in the closure,
                         // so a move-related span is as good as any (and potentially better,
                         // if the overall error is due to a move of the upvar).
-                        let usage_span =
-                            match self.infcx.tcx.typeck(local_did).upvar_capture(upvar_id) {
-                                ty::UpvarCapture::ByValue(Some(span)) => span,
-                                _ => span,
-                            };
+
+                        let usage_span = match captured_place.info.capture_kind {
+                            ty::UpvarCapture::ByValue(Some(span)) => span,
+                            _ => span,
+                        };
                         return Some((*args_span, generator_kind, usage_span));
                     }
                     _ => {}
diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs
index 2f40a90fb55..28f6508cab2 100644
--- a/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs
@@ -510,24 +510,54 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         the_place_err: PlaceRef<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
     ) {
-        let id = id.expect_local();
-        let tables = tcx.typeck(id);
-        let hir_id = tcx.hir().local_def_id_to_hir_id(id);
-        if let Some((span, place)) = tables.closure_kind_origins().get(hir_id) {
-            let reason = if let PlaceBase::Upvar(upvar_id) = place.base {
-                let upvar = ty::place_to_string_for_capture(tcx, place);
-                match tables.upvar_capture(upvar_id) {
-                    ty::UpvarCapture::ByRef(ty::UpvarBorrow {
-                        kind: ty::BorrowKind::MutBorrow | ty::BorrowKind::UniqueImmBorrow,
-                        ..
-                    }) => {
-                        format!("mutable borrow of `{}`", upvar)
-                    }
-                    ty::UpvarCapture::ByValue(_) => {
-                        format!("possible mutation of `{}`", upvar)
+        let closure_local_def_id = id.expect_local();
+        let tables = tcx.typeck(closure_local_def_id);
+        let closure_hir_id = tcx.hir().local_def_id_to_hir_id(closure_local_def_id);
+        if let Some((span, closure_kind_origin)) =
+            &tables.closure_kind_origins().get(closure_hir_id)
+        {
+            let reason = if let PlaceBase::Upvar(upvar_id) = closure_kind_origin.base {
+                let upvar = ty::place_to_string_for_capture(tcx, closure_kind_origin);
+                let root_hir_id = upvar_id.var_path.hir_id;
+                // we have a origin for this closure kind starting at this root variable so it's safe to unwrap here
+                let captured_places = tables.closure_min_captures[id].get(&root_hir_id).unwrap();
+
+                let origin_projection = closure_kind_origin
+                    .projections
+                    .iter()
+                    .map(|proj| proj.kind)
+                    .collect::<Vec<_>>();
+                let mut capture_reason = String::new();
+                for captured_place in captured_places {
+                    let captured_place_kinds = captured_place
+                        .place
+                        .projections
+                        .iter()
+                        .map(|proj| proj.kind)
+                        .collect::<Vec<_>>();
+                    if rustc_middle::ty::is_ancestor_or_same_capture(
+                        &captured_place_kinds,
+                        &origin_projection,
+                    ) {
+                        match captured_place.info.capture_kind {
+                            ty::UpvarCapture::ByRef(ty::UpvarBorrow {
+                                kind: ty::BorrowKind::MutBorrow | ty::BorrowKind::UniqueImmBorrow,
+                                ..
+                            }) => {
+                                capture_reason = format!("mutable borrow of `{}`", upvar);
+                            }
+                            ty::UpvarCapture::ByValue(_) => {
+                                capture_reason = format!("possible mutation of `{}`", upvar);
+                            }
+                            _ => bug!("upvar `{}` borrowed, but not mutably", upvar),
+                        }
+                        break;
                     }
-                    val => bug!("upvar `{}` borrowed, but not mutably: {:?}", upvar, val),
                 }
+                if capture_reason.is_empty() {
+                    bug!("upvar `{}` borrowed, but cannot find reason", upvar);
+                }
+                capture_reason
             } else {
                 bug!("not an upvar")
             };
diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/outlives_suggestion.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/outlives_suggestion.rs
index 7505e6e2dd1..7dc3434bf33 100644
--- a/compiler/rustc_mir/src/borrow_check/diagnostics/outlives_suggestion.rs
+++ b/compiler/rustc_mir/src/borrow_check/diagnostics/outlives_suggestion.rs
@@ -1,4 +1,4 @@
-//! Contains utilities for generating suggestions for borrowck errors related to unsatisified
+//! Contains utilities for generating suggestions for borrowck errors related to unsatisfied
 //! outlives constraints.
 
 use std::collections::BTreeMap;
@@ -157,7 +157,7 @@ impl OutlivesSuggestionBuilder {
         debug!("Collected {:?}: {:?}", fr, outlived_fr);
 
         // Add to set of constraints for final help note.
-        self.constraints_to_add.entry(fr).or_insert(Vec::new()).push(outlived_fr);
+        self.constraints_to_add.entry(fr).or_default().push(outlived_fr);
     }
 
     /// Emit an intermediate note on the given `Diagnostic` if the involved regions are
diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/region_name.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/region_name.rs
index 03738f1b40a..1f168c612f1 100644
--- a/compiler/rustc_mir/src/borrow_check/diagnostics/region_name.rs
+++ b/compiler/rustc_mir/src/borrow_check/diagnostics/region_name.rs
@@ -1,4 +1,5 @@
 use std::fmt::{self, Display};
+use std::iter;
 
 use rustc_errors::DiagnosticBuilder;
 use rustc_hir as hir;
@@ -536,7 +537,8 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                 // just worry about trying to match up the rustc type
                 // with the HIR types:
                 (ty::Tuple(elem_tys), hir::TyKind::Tup(elem_hir_tys)) => {
-                    search_stack.extend(elem_tys.iter().map(|k| k.expect_ty()).zip(*elem_hir_tys));
+                    search_stack
+                        .extend(iter::zip(elem_tys.iter().map(|k| k.expect_ty()), *elem_hir_tys));
                 }
 
                 (ty::Slice(elem_ty), hir::TyKind::Slice(elem_hir_ty))
@@ -611,7 +613,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         args: &'hir hir::GenericArgs<'hir>,
         search_stack: &mut Vec<(Ty<'tcx>, &'hir hir::Ty<'hir>)>,
     ) -> Option<&'hir hir::Lifetime> {
-        for (kind, hir_arg) in substs.iter().zip(args.args) {
+        for (kind, hir_arg) in iter::zip(substs, args.args) {
             match (kind.unpack(), hir_arg) {
                 (GenericArgKind::Lifetime(r), hir::GenericArg::Lifetime(lt)) => {
                     if r.to_region_vid() == needle_fr {
diff --git a/compiler/rustc_mir/src/borrow_check/invalidation.rs b/compiler/rustc_mir/src/borrow_check/invalidation.rs
index 8c05e6fd5d0..e621bafb671 100644
--- a/compiler/rustc_mir/src/borrow_check/invalidation.rs
+++ b/compiler/rustc_mir/src/borrow_check/invalidation.rs
@@ -5,6 +5,7 @@ use rustc_middle::mir::{BorrowKind, Mutability, Operand};
 use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
 use rustc_middle::mir::{Statement, StatementKind};
 use rustc_middle::ty::TyCtxt;
+use std::iter;
 
 use crate::dataflow::indexes::BorrowIndex;
 
@@ -62,14 +63,14 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
 
                 self.mutate_place(location, *lhs, Shallow(None), JustWrite);
             }
-            StatementKind::FakeRead(_, _) => {
+            StatementKind::FakeRead(box (_, _)) => {
                 // Only relevant for initialized/liveness/safety checks.
             }
             StatementKind::SetDiscriminant { place, variant_index: _ } => {
                 self.mutate_place(location, **place, Shallow(None), JustWrite);
             }
             StatementKind::LlvmInlineAsm(asm) => {
-                for (o, output) in asm.asm.outputs.iter().zip(asm.outputs.iter()) {
+                for (o, output) in iter::zip(&asm.asm.outputs, &*asm.outputs) {
                     if o.is_indirect {
                         // FIXME(eddyb) indirect inline asm outputs should
                         // be encoded through MIR place derefs instead.
@@ -92,6 +93,15 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                     self.consume_operand(location, input);
                 }
             }
+            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+                ref src,
+                ref dst,
+                ref count,
+            }) => {
+                self.consume_operand(location, src);
+                self.consume_operand(location, dst);
+                self.consume_operand(location, count);
+            }
             StatementKind::Nop
             | StatementKind::Coverage(..)
             | StatementKind::AscribeUserType(..)
@@ -165,7 +175,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                 self.consume_operand(location, value);
 
                 // Invalidate all borrows of local places
-                let borrow_set = self.borrow_set.clone();
+                let borrow_set = self.borrow_set;
                 let resume = self.location_table.start_index(resume.start_location());
                 for (i, data) in borrow_set.iter_enumerated() {
                     if borrow_of_local_data(data.borrowed_place) {
@@ -177,7 +187,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
             }
             TerminatorKind::Resume | TerminatorKind::Return | TerminatorKind::GeneratorDrop => {
                 // Invalidate all borrows of local places
-                let borrow_set = self.borrow_set.clone();
+                let borrow_set = self.borrow_set;
                 let start = self.location_table.start_index(location);
                 for (i, data) in borrow_set.iter_enumerated() {
                     if borrow_of_local_data(data.borrowed_place) {
@@ -194,8 +204,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
             } => {
                 for op in operands {
                     match *op {
-                        InlineAsmOperand::In { reg: _, ref value }
-                        | InlineAsmOperand::Const { ref value } => {
+                        InlineAsmOperand::In { reg: _, ref value } => {
                             self.consume_operand(location, value);
                         }
                         InlineAsmOperand::Out { reg: _, late: _, place, .. } => {
@@ -209,7 +218,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                                 self.mutate_place(location, out_place, Shallow(None), JustWrite);
                             }
                         }
-                        InlineAsmOperand::SymFn { value: _ }
+                        InlineAsmOperand::Const { value: _ }
+                        | InlineAsmOperand::SymFn { value: _ }
                         | InlineAsmOperand::SymStatic { def_id: _ } => {}
                     }
                 }
@@ -326,8 +336,8 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
                 );
             }
 
-            Rvalue::BinaryOp(_bin_op, ref operand1, ref operand2)
-            | Rvalue::CheckedBinaryOp(_bin_op, ref operand1, ref operand2) => {
+            Rvalue::BinaryOp(_bin_op, box (ref operand1, ref operand2))
+            | Rvalue::CheckedBinaryOp(_bin_op, box (ref operand1, ref operand2)) => {
                 self.consume_operand(location, operand1);
                 self.consume_operand(location, operand2);
             }
@@ -369,7 +379,7 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
         );
         let tcx = self.tcx;
         let body = self.body;
-        let borrow_set = self.borrow_set.clone();
+        let borrow_set = self.borrow_set;
         let indices = self.borrow_set.indices();
         each_borrow_involving_path(
             self,
@@ -377,7 +387,7 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
             body,
             location,
             (sd, place),
-            &borrow_set.clone(),
+            borrow_set,
             indices,
             |this, borrow_index, borrow| {
                 match (rw, borrow.kind) {
diff --git a/compiler/rustc_mir/src/borrow_check/mod.rs b/compiler/rustc_mir/src/borrow_check/mod.rs
index 375d4649171..2d1d83b1655 100644
--- a/compiler/rustc_mir/src/borrow_check/mod.rs
+++ b/compiler/rustc_mir/src/borrow_check/mod.rs
@@ -25,6 +25,7 @@ use either::Either;
 use smallvec::SmallVec;
 use std::cell::RefCell;
 use std::collections::BTreeMap;
+use std::iter;
 use std::mem;
 use std::rc::Rc;
 
@@ -573,7 +574,7 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
 
                 self.mutate_place(location, (*lhs, span), Shallow(None), JustWrite, flow_state);
             }
-            StatementKind::FakeRead(_, box ref place) => {
+            StatementKind::FakeRead(box (_, ref place)) => {
                 // Read for match doesn't access any memory and is used to
                 // assert that a place is safe and live. So we don't have to
                 // do any checks here.
@@ -595,7 +596,7 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
                 self.mutate_place(location, (**place, span), Shallow(None), JustWrite, flow_state);
             }
             StatementKind::LlvmInlineAsm(ref asm) => {
-                for (o, output) in asm.asm.outputs.iter().zip(asm.outputs.iter()) {
+                for (o, output) in iter::zip(&asm.asm.outputs, &*asm.outputs) {
                     if o.is_indirect {
                         // FIXME(eddyb) indirect inline asm outputs should
                         // be encoded through MIR place derefs instead.
@@ -626,6 +627,15 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
                     self.consume_operand(location, (input, span), flow_state);
                 }
             }
+
+            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+                ..
+            }) => {
+                span_bug!(
+                    span,
+                    "Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
+                )
+            }
             StatementKind::Nop
             | StatementKind::Coverage(..)
             | StatementKind::AscribeUserType(..)
@@ -724,8 +734,7 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
             } => {
                 for op in operands {
                     match *op {
-                        InlineAsmOperand::In { reg: _, ref value }
-                        | InlineAsmOperand::Const { ref value } => {
+                        InlineAsmOperand::In { reg: _, ref value } => {
                             self.consume_operand(loc, (value, span), flow_state);
                         }
                         InlineAsmOperand::Out { reg: _, late: _, place, .. } => {
@@ -751,7 +760,8 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
                                 );
                             }
                         }
-                        InlineAsmOperand::SymFn { value: _ }
+                        InlineAsmOperand::Const { value: _ }
+                        | InlineAsmOperand::SymFn { value: _ }
                         | InlineAsmOperand::SymStatic { def_id: _ } => {}
                     }
                 }
@@ -1316,8 +1326,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 );
             }
 
-            Rvalue::BinaryOp(_bin_op, ref operand1, ref operand2)
-            | Rvalue::CheckedBinaryOp(_bin_op, ref operand1, ref operand2) => {
+            Rvalue::BinaryOp(_bin_op, box (ref operand1, ref operand2))
+            | Rvalue::CheckedBinaryOp(_bin_op, box (ref operand1, ref operand2)) => {
                 self.consume_operand(location, (operand1, span), flow_state);
                 self.consume_operand(location, (operand2, span), flow_state);
             }
diff --git a/compiler/rustc_mir/src/borrow_check/places_conflict.rs b/compiler/rustc_mir/src/borrow_check/places_conflict.rs
index 02c7b7dc200..3654b51949e 100644
--- a/compiler/rustc_mir/src/borrow_check/places_conflict.rs
+++ b/compiler/rustc_mir/src/borrow_check/places_conflict.rs
@@ -5,6 +5,7 @@ use rustc_hir as hir;
 use rustc_middle::mir::{Body, BorrowKind, Local, Place, PlaceElem, PlaceRef, ProjectionElem};
 use rustc_middle::ty::{self, TyCtxt};
 use std::cmp::max;
+use std::iter;
 
 /// When checking if a place conflicts with another place, this enum is used to influence decisions
 /// where a place might be equal or disjoint with another place, such as if `a[i] == a[j]`.
@@ -139,7 +140,7 @@ fn place_components_conflict<'tcx>(
 
     // loop invariant: borrow_c is always either equal to access_c or disjoint from it.
     for (i, (borrow_c, &access_c)) in
-        borrow_place.projection.iter().zip(access_place.projection.iter()).enumerate()
+        iter::zip(borrow_place.projection, access_place.projection).enumerate()
     {
         debug!("borrow_conflicts_with_place: borrow_c = {:?}", borrow_c);
         let borrow_proj_base = &borrow_place.projection[..i];
diff --git a/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs b/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs
index a272e922a50..7156612f473 100644
--- a/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs
+++ b/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs
@@ -1,5 +1,5 @@
 //! This module provides linkage between RegionInferenceContext and
-//! librustc_graphviz traits, specialized to attaching borrowck analysis
+//! `rustc_graphviz` traits, specialized to attaching borrowck analysis
 //! data to rendered labels.
 
 use std::borrow::Cow;
diff --git a/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs b/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs
index f7c902355cb..0d1d2551042 100644
--- a/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs
+++ b/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs
@@ -47,6 +47,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// Calling `universal_upper_bound` for such a region gives `fr_fn_body`,
     /// which has no `external_name` in which case we use `'empty` as the
     /// region to pass to `infer_opaque_definition_from_instantiation`.
+    #[instrument(skip(self, infcx))]
     pub(in crate::borrow_check) fn infer_opaque_types(
         &self,
         infcx: &InferCtxt<'_, 'tcx>,
@@ -56,10 +57,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         opaque_ty_decls
             .into_iter()
             .map(|(opaque_def_id, ty::ResolvedOpaqueTy { concrete_type, substs })| {
-                debug!(
-                    "infer_opaque_types(concrete_type = {:?}, substs = {:?})",
-                    concrete_type, substs
-                );
+                debug!(?concrete_type, ?substs);
 
                 let mut subst_regions = vec![self.universal_regions.fr_static];
                 let universal_substs =
@@ -110,10 +108,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                         }
                     });
 
-                debug!(
-                    "infer_opaque_types(universal_concrete_type = {:?}, universal_substs = {:?})",
-                    universal_concrete_type, universal_substs
-                );
+                debug!(?universal_concrete_type, ?universal_substs);
 
                 let remapped_type = infcx.infer_opaque_definition_from_instantiation(
                     opaque_def_id,
diff --git a/compiler/rustc_mir/src/borrow_check/type_check/input_output.rs b/compiler/rustc_mir/src/borrow_check/type_check/input_output.rs
index 77d91366224..1bb447d1057 100644
--- a/compiler/rustc_mir/src/borrow_check/type_check/input_output.rs
+++ b/compiler/rustc_mir/src/borrow_check/type_check/input_output.rs
@@ -70,6 +70,12 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
 
         // Equate expected input tys with those in the MIR.
         for (argument_index, &normalized_input_ty) in normalized_input_tys.iter().enumerate() {
+            if argument_index + 1 >= body.local_decls.len() {
+                self.tcx()
+                    .sess
+                    .delay_span_bug(body.span, "found more normalized_input_ty than local_decls");
+                break;
+            }
             // In MIR, argument N is stored in local N+1.
             let local = Local::new(argument_index + 1);
 
diff --git a/compiler/rustc_mir/src/borrow_check/type_check/mod.rs b/compiler/rustc_mir/src/borrow_check/type_check/mod.rs
index fa5b342c6e2..3248554e204 100644
--- a/compiler/rustc_mir/src/borrow_check/type_check/mod.rs
+++ b/compiler/rustc_mir/src/borrow_check/type_check/mod.rs
@@ -282,7 +282,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
 
     fn visit_constant(&mut self, constant: &Constant<'tcx>, location: Location) {
         self.super_constant(constant, location);
-        let ty = self.sanitize_type(constant, constant.literal.ty);
+        let ty = self.sanitize_type(constant, constant.literal.ty());
 
         self.cx.infcx.tcx.for_each_free_region(&ty, |live_region| {
             let live_region_vid =
@@ -296,7 +296,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
 
         if let Some(annotation_index) = constant.user_ty {
             if let Err(terr) = self.cx.relate_type_and_user_type(
-                constant.literal.ty,
+                constant.literal.ty(),
                 ty::Variance::Invariant,
                 &UserTypeProjection { base: annotation_index, projs: vec![] },
                 location.to_locations(),
@@ -308,13 +308,20 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                     constant,
                     "bad constant user type {:?} vs {:?}: {:?}",
                     annotation,
-                    constant.literal.ty,
+                    constant.literal.ty(),
                     terr,
                 );
             }
         } else {
             let tcx = self.tcx();
-            if let ty::ConstKind::Unevaluated(def, substs, promoted) = constant.literal.val {
+            let maybe_uneval = match constant.literal {
+                ConstantKind::Ty(ct) => match ct.val {
+                    ty::ConstKind::Unevaluated(uv) => Some(uv),
+                    _ => None,
+                },
+                _ => None,
+            };
+            if let Some(ty::Unevaluated { def, substs, promoted }) = maybe_uneval {
                 if let Some(promoted) = promoted {
                     let check_err = |verifier: &mut TypeVerifier<'a, 'b, 'tcx>,
                                      promoted: &Body<'tcx>,
@@ -349,7 +356,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                         location.to_locations(),
                         ConstraintCategory::Boring,
                         self.cx.param_env.and(type_op::ascribe_user_type::AscribeUserType::new(
-                            constant.literal.ty,
+                            constant.literal.ty(),
                             def.did,
                             UserSubsts { substs, user_self_ty: None },
                         )),
@@ -367,7 +374,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                 let unnormalized_ty = tcx.type_of(static_def_id);
                 let locations = location.to_locations();
                 let normalized_ty = self.cx.normalize(unnormalized_ty, locations);
-                let literal_ty = constant.literal.ty.builtin_deref(true).unwrap().ty;
+                let literal_ty = constant.literal.ty().builtin_deref(true).unwrap().ty;
 
                 if let Err(terr) = self.cx.eq_types(
                     normalized_ty,
@@ -379,7 +386,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                 }
             }
 
-            if let ty::FnDef(def_id, substs) = *constant.literal.ty.kind() {
+            if let ty::FnDef(def_id, substs) = *constant.literal.ty().kind() {
                 let instantiated_predicates = tcx.predicates_of(def_id).instantiate(tcx, substs);
                 self.cx.normalize_and_prove_instantiated_predicates(
                     instantiated_predicates,
@@ -1520,6 +1527,12 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     );
                 }
             }
+            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+                ..
+            }) => span_bug!(
+                stmt.source_info.span,
+                "Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics",
+            ),
             StatementKind::FakeRead(..)
             | StatementKind::StorageLive(..)
             | StatementKind::StorageDead(..)
@@ -1757,7 +1770,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         if args.len() < sig.inputs().len() || (args.len() > sig.inputs().len() && !sig.c_variadic) {
             span_mirbug!(self, term, "call to {:?} with wrong # of args", sig);
         }
-        for (n, (fn_arg, op_arg)) in sig.inputs().iter().zip(args).enumerate() {
+        for (n, (fn_arg, op_arg)) in iter::zip(sig.inputs(), args).enumerate() {
             let op_arg_ty = op_arg.ty(body, self.tcx());
             let op_arg_ty = self.normalize(op_arg_ty, term_location);
             let category = if from_hir_call {
@@ -2015,7 +2028,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                                             traits::ObligationCauseCode::RepeatVec(is_const_fn),
                                         ),
                                         self.param_env,
-                                        ty::Binder::bind(ty::TraitRef::new(
+                                        ty::Binder::dummy(ty::TraitRef::new(
                                             self.tcx().require_lang_item(
                                                 LangItem::Copy,
                                                 Some(self.last_span),
@@ -2299,8 +2312,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
 
             Rvalue::BinaryOp(
                 BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge,
-                left,
-                right,
+                box (left, right),
             ) => {
                 let ty_left = left.ty(body, tcx);
                 match ty_left.kind() {
diff --git a/compiler/rustc_mir/src/borrow_check/universal_regions.rs b/compiler/rustc_mir/src/borrow_check/universal_regions.rs
index 4b1acc1cd10..c2ac1e289ce 100644
--- a/compiler/rustc_mir/src/borrow_check/universal_regions.rs
+++ b/compiler/rustc_mir/src/borrow_check/universal_regions.rs
@@ -580,7 +580,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
 
         let global_mapping = iter::once((tcx.lifetimes.re_static, fr_static));
         let subst_mapping =
-            identity_substs.regions().zip(fr_substs.regions().map(|r| r.to_region_vid()));
+            iter::zip(identity_substs.regions(), fr_substs.regions().map(|r| r.to_region_vid()));
 
         UniversalRegionIndices { indices: global_mapping.chain(subst_mapping).collect() }
     }
@@ -589,31 +589,45 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         &self,
         indices: &UniversalRegionIndices<'tcx>,
         defining_ty: DefiningTy<'tcx>,
-    ) -> ty::Binder<&'tcx ty::List<Ty<'tcx>>> {
+    ) -> ty::Binder<'tcx, &'tcx ty::List<Ty<'tcx>>> {
         let tcx = self.infcx.tcx;
         match defining_ty {
             DefiningTy::Closure(def_id, substs) => {
                 assert_eq!(self.mir_def.did.to_def_id(), def_id);
                 let closure_sig = substs.as_closure().sig();
                 let inputs_and_output = closure_sig.inputs_and_output();
-                let closure_ty = tcx.closure_env_ty(def_id, substs).unwrap();
-                ty::Binder::fuse(closure_ty, inputs_and_output, |closure_ty, inputs_and_output| {
-                    // The "inputs" of the closure in the
-                    // signature appear as a tuple.  The MIR side
-                    // flattens this tuple.
-                    let (&output, tuplized_inputs) = inputs_and_output.split_last().unwrap();
-                    assert_eq!(tuplized_inputs.len(), 1, "multiple closure inputs");
-                    let inputs = match tuplized_inputs[0].kind() {
-                        ty::Tuple(inputs) => inputs,
-                        _ => bug!("closure inputs not a tuple: {:?}", tuplized_inputs[0]),
-                    };
+                let bound_vars = tcx.mk_bound_variable_kinds(
+                    inputs_and_output
+                        .bound_vars()
+                        .iter()
+                        .chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
+                );
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(bound_vars.len() - 1),
+                    kind: ty::BrEnv,
+                };
+                let env_region = ty::ReLateBound(ty::INNERMOST, br);
+                let closure_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap();
+
+                // The "inputs" of the closure in the
+                // signature appear as a tuple.  The MIR side
+                // flattens this tuple.
+                let (&output, tuplized_inputs) =
+                    inputs_and_output.skip_binder().split_last().unwrap();
+                assert_eq!(tuplized_inputs.len(), 1, "multiple closure inputs");
+                let inputs = match tuplized_inputs[0].kind() {
+                    ty::Tuple(inputs) => inputs,
+                    _ => bug!("closure inputs not a tuple: {:?}", tuplized_inputs[0]),
+                };
 
+                ty::Binder::bind_with_vars(
                     tcx.mk_type_list(
                         iter::once(closure_ty)
                             .chain(inputs.iter().map(|k| k.expect_ty()))
                             .chain(iter::once(output)),
-                    )
-                })
+                    ),
+                    bound_vars,
+                )
             }
 
             DefiningTy::Generator(def_id, substs, movability) => {
@@ -657,7 +671,7 @@ trait InferCtxtExt<'tcx> {
         &self,
         origin: NllRegionVariableOrigin,
         all_outlive_scope: LocalDefId,
-        value: ty::Binder<T>,
+        value: ty::Binder<'tcx, T>,
         indices: &mut UniversalRegionIndices<'tcx>,
     ) -> T
     where
@@ -686,7 +700,7 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'cx, 'tcx> {
         &self,
         origin: NllRegionVariableOrigin,
         all_outlive_scope: LocalDefId,
-        value: ty::Binder<T>,
+        value: ty::Binder<'tcx, T>,
         indices: &mut UniversalRegionIndices<'tcx>,
     ) -> T
     where
diff --git a/compiler/rustc_mir/src/const_eval/eval_queries.rs b/compiler/rustc_mir/src/const_eval/eval_queries.rs
index fa234ff5feb..d51adc8864d 100644
--- a/compiler/rustc_mir/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_mir/src/const_eval/eval_queries.rs
@@ -5,6 +5,7 @@ use crate::interpret::{
     Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, Scalar,
     ScalarMaybeUninit, StackPopCleanup,
 };
+use crate::util::pretty::display_allocation;
 
 use rustc_errors::ErrorReported;
 use rustc_hir::def::DefKind;
@@ -360,6 +361,15 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
                     "it is undefined behavior to use this value",
                     |mut diag| {
                         diag.note(note_on_undefined_behavior_error());
+                        diag.note(&format!(
+                            "the raw bytes of the constant ({}",
+                            display_allocation(
+                                *ecx.tcx,
+                                ecx.tcx
+                                    .global_alloc(mplace.ptr.assert_ptr().alloc_id)
+                                    .unwrap_memory()
+                            )
+                        ));
                         diag.emit();
                     },
                 ))
diff --git a/compiler/rustc_mir/src/const_eval/machine.rs b/compiler/rustc_mir/src/const_eval/machine.rs
index 61785a52729..8e9148f5b66 100644
--- a/compiler/rustc_mir/src/const_eval/machine.rs
+++ b/compiler/rustc_mir/src/const_eval/machine.rs
@@ -53,7 +53,7 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
 /// Extra machine state for CTFE, and the Machine instance
 pub struct CompileTimeInterpreter<'mir, 'tcx> {
     /// For now, the number of terminators that can be evaluated before we throw a resource
-    /// exhuastion error.
+    /// exhaustion error.
     ///
     /// Setting this to `0` disables the limit and allows the interpreter to run forever.
     pub steps_remaining: usize,
diff --git a/compiler/rustc_mir/src/const_eval/mod.rs b/compiler/rustc_mir/src/const_eval/mod.rs
index a4e1cd2faa3..3f14efc920f 100644
--- a/compiler/rustc_mir/src/const_eval/mod.rs
+++ b/compiler/rustc_mir/src/const_eval/mod.rs
@@ -3,12 +3,15 @@
 use std::convert::TryFrom;
 
 use rustc_hir::Mutability;
-use rustc_middle::mir;
 use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::{
+    mir::{self, interpret::ConstAlloc},
+    ty::ScalarInt,
+};
 use rustc_span::{source_map::DUMMY_SP, symbol::Symbol};
 
 use crate::interpret::{
-    intern_const_alloc_recursive, ConstValue, InternKind, InterpCx, MemPlaceMeta, Scalar,
+    intern_const_alloc_recursive, ConstValue, InternKind, InterpCx, MPlaceTy, MemPlaceMeta, Scalar,
 };
 
 mod error;
@@ -35,6 +38,100 @@ pub(crate) fn const_caller_location(
     ConstValue::Scalar(loc_place.ptr)
 }
 
+/// Convert an evaluated constant to a type level constant
+pub(crate) fn const_to_valtree<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    raw: ConstAlloc<'tcx>,
+) -> Option<ty::ValTree<'tcx>> {
+    let ecx = mk_eval_cx(
+        tcx, DUMMY_SP, param_env,
+        // It is absolutely crucial for soundness that
+        // we do not read from static items or other mutable memory.
+        false,
+    );
+    let place = ecx.raw_const_to_mplace(raw).unwrap();
+    const_to_valtree_inner(&ecx, &place)
+}
+
+fn const_to_valtree_inner<'tcx>(
+    ecx: &CompileTimeEvalContext<'tcx, 'tcx>,
+    place: &MPlaceTy<'tcx>,
+) -> Option<ty::ValTree<'tcx>> {
+    let branches = |n, variant| {
+        let place = match variant {
+            Some(variant) => ecx.mplace_downcast(&place, variant).unwrap(),
+            None => *place,
+        };
+        let variant =
+            variant.map(|variant| Some(ty::ValTree::Leaf(ScalarInt::from(variant.as_u32()))));
+        let fields = (0..n).map(|i| {
+            let field = ecx.mplace_field(&place, i).unwrap();
+            const_to_valtree_inner(ecx, &field)
+        });
+        // For enums, we preped their variant index before the variant's fields so we can figure out
+        // the variant again when just seeing a valtree.
+        let branches = variant.into_iter().chain(fields);
+        Some(ty::ValTree::Branch(
+            ecx.tcx.arena.alloc_from_iter(branches.collect::<Option<Vec<_>>>()?),
+        ))
+    };
+    match place.layout.ty.kind() {
+        ty::FnDef(..) => Some(ty::ValTree::zst()),
+        ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => {
+            let val = ecx.read_immediate(&place.into()).unwrap();
+            let val = val.to_scalar().unwrap();
+            Some(ty::ValTree::Leaf(val.assert_int()))
+        }
+
+        // Raw pointers are not allowed in type level constants, as we cannot properly test them for
+        // equality at compile-time (see `ptr_guaranteed_eq`/`_ne`).
+        // Technically we could allow function pointers (represented as `ty::Instance`), but this is not guaranteed to
+        // agree with runtime equality tests.
+        ty::FnPtr(_) | ty::RawPtr(_) => None,
+        ty::Ref(..) => unimplemented!("need to use deref_const"),
+
+        // Trait objects are not allowed in type level constants, as we have no concept for
+        // resolving their backing type, even if we can do that at const eval time. We may
+        // hypothetically be able to allow `dyn StructuralEq` trait objects in the future,
+        // but it is unclear if this is useful.
+        ty::Dynamic(..) => None,
+
+        ty::Slice(_) | ty::Str => {
+            unimplemented!("need to find the backing data of the slice/str and recurse on that")
+        }
+        ty::Tuple(substs) => branches(substs.len(), None),
+        ty::Array(_, len) => branches(usize::try_from(len.eval_usize(ecx.tcx.tcx, ecx.param_env)).unwrap(), None),
+
+        ty::Adt(def, _) => {
+            if def.variants.is_empty() {
+                bug!("uninhabited types should have errored and never gotten converted to valtree")
+            }
+
+            let variant = ecx.read_discriminant(&place.into()).unwrap().1;
+
+            branches(def.variants[variant].fields.len(), def.is_enum().then_some(variant))
+        }
+
+        ty::Never
+        | ty::Error(_)
+        | ty::Foreign(..)
+        | ty::Infer(ty::FreshIntTy(_))
+        | ty::Infer(ty::FreshFloatTy(_))
+        | ty::Projection(..)
+        | ty::Param(_)
+        | ty::Bound(..)
+        | ty::Placeholder(..)
+        // FIXME(oli-obk): we could look behind opaque types
+        | ty::Opaque(..)
+        | ty::Infer(_)
+        // FIXME(oli-obk): we can probably encode closures just like structs
+        | ty::Closure(..)
+        | ty::Generator(..)
+        | ty::GeneratorWitness(..) => None,
+    }
+}
+
 /// This function uses `unwrap` copiously, because an already validated constant
 /// must have valid fields and can thus never fail outside of compiler bugs. However, it is
 /// invoked from the pretty printer, where it can receive enums with no variants and e.g.
diff --git a/compiler/rustc_mir/src/dataflow/framework/cursor.rs b/compiler/rustc_mir/src/dataflow/framework/cursor.rs
index 4942bed656c..c000e49c14b 100644
--- a/compiler/rustc_mir/src/dataflow/framework/cursor.rs
+++ b/compiler/rustc_mir/src/dataflow/framework/cursor.rs
@@ -64,10 +64,6 @@ where
         }
     }
 
-    pub fn body(&self) -> &'mir mir::Body<'tcx> {
-        self.body
-    }
-
     /// Returns the underlying `Results`.
     pub fn results(&self) -> &Results<'tcx, A> {
         &self.results.borrow()
diff --git a/compiler/rustc_mir/src/dataflow/framework/lattice.rs b/compiler/rustc_mir/src/dataflow/framework/lattice.rs
index e7ef9267db5..f937b31f4cf 100644
--- a/compiler/rustc_mir/src/dataflow/framework/lattice.rs
+++ b/compiler/rustc_mir/src/dataflow/framework/lattice.rs
@@ -40,6 +40,7 @@
 
 use rustc_index::bit_set::BitSet;
 use rustc_index::vec::{Idx, IndexVec};
+use std::iter;
 
 /// A [partially ordered set][poset] that has a [least upper bound][lub] for any pair of elements
 /// in the set.
@@ -110,7 +111,7 @@ impl<I: Idx, T: JoinSemiLattice> JoinSemiLattice for IndexVec<I, T> {
         assert_eq!(self.len(), other.len());
 
         let mut changed = false;
-        for (a, b) in self.iter_mut().zip(other.iter()) {
+        for (a, b) in iter::zip(self, other) {
             changed |= a.join(b);
         }
         changed
@@ -122,7 +123,7 @@ impl<I: Idx, T: MeetSemiLattice> MeetSemiLattice for IndexVec<I, T> {
         assert_eq!(self.len(), other.len());
 
         let mut changed = false;
-        for (a, b) in self.iter_mut().zip(other.iter()) {
+        for (a, b) in iter::zip(self, other) {
             changed |= a.meet(b);
         }
         changed
diff --git a/compiler/rustc_mir/src/dataflow/framework/mod.rs b/compiler/rustc_mir/src/dataflow/framework/mod.rs
index 3f7808c2090..344d7b9becd 100644
--- a/compiler/rustc_mir/src/dataflow/framework/mod.rs
+++ b/compiler/rustc_mir/src/dataflow/framework/mod.rs
@@ -510,7 +510,7 @@ impl EffectIndex {
         }
     }
 
-    /// Returns `true` if the effect at `self` should be applied eariler than the effect at `other`
+    /// Returns `true` if the effect at `self` should be applied earlier than the effect at `other`
     /// in forward order.
     fn precedes_in_forward_order(self, other: Self) -> bool {
         let ord = self
diff --git a/compiler/rustc_mir/src/dataflow/impls/borrows.rs b/compiler/rustc_mir/src/dataflow/impls/borrows.rs
index b149ffa9667..c92cff1433f 100644
--- a/compiler/rustc_mir/src/dataflow/impls/borrows.rs
+++ b/compiler/rustc_mir/src/dataflow/impls/borrows.rs
@@ -11,6 +11,7 @@ use crate::borrow_check::{
 use crate::dataflow::{self, fmt::DebugWithContext, GenKill};
 
 use std::fmt;
+use std::iter;
 
 rustc_index::newtype_index! {
     pub struct BorrowIndex {
@@ -292,7 +293,7 @@ impl<'tcx> dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
             }
 
             mir::StatementKind::LlvmInlineAsm(ref asm) => {
-                for (output, kind) in asm.outputs.iter().zip(&asm.asm.outputs) {
+                for (output, kind) in iter::zip(&*asm.outputs, &asm.asm.outputs) {
                     if !kind.is_indirect && !kind.is_rw {
                         self.kill_borrows_on_place(trans, *output);
                     }
@@ -305,6 +306,7 @@ impl<'tcx> dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::Coverage(..)
+            | mir::StatementKind::CopyNonOverlapping(..)
             | mir::StatementKind::Nop => {}
         }
     }
diff --git a/compiler/rustc_mir/src/dataflow/impls/storage_liveness.rs b/compiler/rustc_mir/src/dataflow/impls/storage_liveness.rs
index 9250cd40847..792664597fd 100644
--- a/compiler/rustc_mir/src/dataflow/impls/storage_liveness.rs
+++ b/compiler/rustc_mir/src/dataflow/impls/storage_liveness.rs
@@ -149,6 +149,7 @@ impl<'mir, 'tcx> dataflow::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir,
             | StatementKind::FakeRead(..)
             | StatementKind::Nop
             | StatementKind::Retag(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::StorageLive(..) => {}
         }
     }
diff --git a/compiler/rustc_mir/src/dataflow/move_paths/builder.rs b/compiler/rustc_mir/src/dataflow/move_paths/builder.rs
index ee78ff00c9b..994b403abf3 100644
--- a/compiler/rustc_mir/src/dataflow/move_paths/builder.rs
+++ b/compiler/rustc_mir/src/dataflow/move_paths/builder.rs
@@ -4,6 +4,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
 use smallvec::{smallvec, SmallVec};
 
+use std::iter;
 use std::mem;
 
 use super::abs_domain::Lift;
@@ -292,11 +293,11 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                 }
                 self.gather_rvalue(rval);
             }
-            StatementKind::FakeRead(_, place) => {
-                self.create_move_path(**place);
+            StatementKind::FakeRead(box (_, place)) => {
+                self.create_move_path(*place);
             }
             StatementKind::LlvmInlineAsm(ref asm) => {
-                for (output, kind) in asm.outputs.iter().zip(&asm.asm.outputs) {
+                for (output, kind) in iter::zip(&*asm.outputs, &asm.asm.outputs) {
                     if !kind.is_indirect {
                         self.gather_init(output.as_ref(), InitKind::Deep);
                     }
@@ -318,6 +319,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             StatementKind::Retag { .. }
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::Nop => {}
         }
     }
@@ -329,8 +331,8 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             | Rvalue::Repeat(ref operand, _)
             | Rvalue::Cast(_, ref operand, _)
             | Rvalue::UnaryOp(_, ref operand) => self.gather_operand(operand),
-            Rvalue::BinaryOp(ref _binop, ref lhs, ref rhs)
-            | Rvalue::CheckedBinaryOp(ref _binop, ref lhs, ref rhs) => {
+            Rvalue::BinaryOp(ref _binop, box (ref lhs, ref rhs))
+            | Rvalue::CheckedBinaryOp(ref _binop, box (ref lhs, ref rhs)) => {
                 self.gather_operand(lhs);
                 self.gather_operand(rhs);
             }
@@ -423,7 +425,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                 for op in operands {
                     match *op {
                         InlineAsmOperand::In { reg: _, ref value }
-                        | InlineAsmOperand::Const { ref value } => {
+                         => {
                             self.gather_operand(value);
                         }
                         InlineAsmOperand::Out { reg: _, late: _, place, .. } => {
@@ -439,7 +441,8 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                                 self.gather_init(out_place.as_ref(), InitKind::Deep);
                             }
                         }
-                        InlineAsmOperand::SymFn { value: _ }
+                        InlineAsmOperand::Const { value: _ }
+                        | InlineAsmOperand::SymFn { value: _ }
                         | InlineAsmOperand::SymStatic { def_id: _ } => {}
                     }
                 }
diff --git a/compiler/rustc_mir/src/interpret/eval_context.rs b/compiler/rustc_mir/src/interpret/eval_context.rs
index 6b796eb3721..2d83d6cfbdc 100644
--- a/compiler/rustc_mir/src/interpret/eval_context.rs
+++ b/compiler/rustc_mir/src/interpret/eval_context.rs
@@ -232,6 +232,8 @@ impl<'mir, 'tcx, Tag, Extra> Frame<'mir, 'tcx, Tag, Extra> {
     /// this frame (can happen e.g. during frame initialization, and during unwinding on
     /// frames without cleanup code).
     /// We basically abuse `Result` as `Either`.
+    ///
+    /// Used by priroda.
     pub fn current_loc(&self) -> Result<mir::Location, Span> {
         self.loc
     }
@@ -460,11 +462,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     }
 
     #[inline]
-    pub fn type_is_sized(&self, ty: Ty<'tcx>) -> bool {
-        ty.is_sized(self.tcx, self.param_env)
-    }
-
-    #[inline]
     pub fn type_is_freeze(&self, ty: Ty<'tcx>) -> bool {
         ty.is_freeze(self.tcx, self.param_env)
     }
@@ -527,6 +524,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         }
     }
 
+    #[inline(always)]
     pub fn layout_of_local(
         &self,
         frame: &Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>,
@@ -689,7 +687,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             let span = const_.span;
             let const_ =
                 self.subst_from_current_frame_and_normalize_erasing_regions(const_.literal);
-            self.const_to_op(const_, None).map_err(|err| {
+            self.mir_const_to_op(&const_, None).map_err(|err| {
                 // If there was an error, set the span of the current frame to this constant.
                 // Avoiding doing this when evaluation succeeds.
                 self.frame_mut().loc = Err(span);
diff --git a/compiler/rustc_mir/src/interpret/intrinsics.rs b/compiler/rustc_mir/src/interpret/intrinsics.rs
index d36b3a7d9b5..d74ef66a4b2 100644
--- a/compiler/rustc_mir/src/interpret/intrinsics.rs
+++ b/compiler/rustc_mir/src/interpret/intrinsics.rs
@@ -171,8 +171,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 };
                 let val =
                     self.tcx.const_eval_global_id(self.param_env, gid, Some(self.tcx.span))?;
-                let const_ = ty::Const { val: ty::ConstKind::Value(val), ty };
-                let val = self.const_to_op(&const_, None)?;
+                let val = self.const_val_to_op(val, ty, Some(dest.layout))?;
                 self.copy_op(&val, dest)?;
             }
 
@@ -323,28 +322,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let result = Scalar::from_uint(truncated_bits, layout.size);
                 self.write_scalar(result, dest)?;
             }
-            sym::copy | sym::copy_nonoverlapping => {
-                let elem_ty = instance.substs.type_at(0);
-                let elem_layout = self.layout_of(elem_ty)?;
-                let count = self.read_scalar(&args[2])?.to_machine_usize(self)?;
-                let elem_align = elem_layout.align.abi;
-
-                let size = elem_layout.size.checked_mul(count, self).ok_or_else(|| {
-                    err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
-                })?;
-                let src = self.read_scalar(&args[0])?.check_init()?;
-                let src = self.memory.check_ptr_access(src, size, elem_align)?;
-                let dest = self.read_scalar(&args[1])?.check_init()?;
-                let dest = self.memory.check_ptr_access(dest, size, elem_align)?;
-
-                if let (Some(src), Some(dest)) = (src, dest) {
-                    self.memory.copy(
-                        src,
-                        dest,
-                        size,
-                        intrinsic_name == sym::copy_nonoverlapping,
-                    )?;
-                }
+            sym::copy => {
+                self.copy(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
             }
             sym::offset => {
                 let ptr = self.read_scalar(&args[0])?.check_init()?;
diff --git a/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs b/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs
index e1ec4cc5e97..ae5e78ee33f 100644
--- a/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs
+++ b/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs
@@ -74,7 +74,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> {
 
     fn print_dyn_existential(
         mut self,
-        predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     ) -> Result<Self::DynExistential, Self::Error> {
         let mut first = true;
         for p in predicates {
diff --git a/compiler/rustc_mir/src/interpret/memory.rs b/compiler/rustc_mir/src/interpret/memory.rs
index f3e373813ca..fe5ebf0b6fe 100644
--- a/compiler/rustc_mir/src/interpret/memory.rs
+++ b/compiler/rustc_mir/src/interpret/memory.rs
@@ -854,7 +854,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
             Some(ptr) => ptr,
             None => {
                 // zero-sized access
-                src.next().expect_none("iterator said it was empty but returned an element");
+                assert_matches!(
+                    src.next(),
+                    None,
+                    "iterator said it was empty but returned an element"
+                );
                 return Ok(());
             }
         };
@@ -880,7 +884,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
             Some(ptr) => ptr,
             None => {
                 // zero-sized access
-                src.next().expect_none("iterator said it was empty but returned an element");
+                assert_matches!(
+                    src.next(),
+                    None,
+                    "iterator said it was empty but returned an element"
+                );
                 return Ok(());
             }
         };
@@ -894,7 +902,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
             let offset_ptr = ptr.offset(Size::from_bytes(idx) * 2, &tcx)?; // `Size` multiplication
             allocation.write_scalar(&tcx, offset_ptr, val.into(), Size::from_bytes(2))?;
         }
-        src.next().expect_none("iterator was longer than it said it would be");
+        assert_matches!(src.next(), None, "iterator was longer than it said it would be");
         Ok(())
     }
 
diff --git a/compiler/rustc_mir/src/interpret/operand.rs b/compiler/rustc_mir/src/interpret/operand.rs
index 901ed6809f2..e5bc9320260 100644
--- a/compiler/rustc_mir/src/interpret/operand.rs
+++ b/compiler/rustc_mir/src/interpret/operand.rs
@@ -32,7 +32,7 @@ pub enum Immediate<Tag = ()> {
     ScalarPair(ScalarMaybeUninit<Tag>, ScalarMaybeUninit<Tag>),
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(Immediate, 56);
 
 impl<Tag> From<ScalarMaybeUninit<Tag>> for Immediate<Tag> {
@@ -77,14 +77,6 @@ impl<'tcx, Tag> Immediate<Tag> {
     pub fn to_scalar(self) -> InterpResult<'tcx, Scalar<Tag>> {
         self.to_scalar_or_uninit().check_init()
     }
-
-    #[inline]
-    pub fn to_scalar_pair(self) -> InterpResult<'tcx, (Scalar<Tag>, Scalar<Tag>)> {
-        match self {
-            Immediate::Scalar(..) => bug!("Got a thin pointer where a scalar pair was expected"),
-            Immediate::ScalarPair(a, b) => Ok((a.check_init()?, b.check_init()?)),
-        }
-    }
 }
 
 // ScalarPair needs a type to interpret, so we often have an immediate and a type together
@@ -95,7 +87,7 @@ pub struct ImmTy<'tcx, Tag = ()> {
     pub layout: TyAndLayout<'tcx>,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(ImmTy<'_>, 72);
 
 impl<Tag: Copy> std::fmt::Display for ImmTy<'tcx, Tag> {
@@ -162,7 +154,7 @@ pub struct OpTy<'tcx, Tag = ()> {
     pub layout: TyAndLayout<'tcx>,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(OpTy<'_, ()>, 80);
 
 impl<'tcx, Tag> std::ops::Deref for OpTy<'tcx, Tag> {
@@ -233,7 +225,7 @@ impl<'tcx, Tag: Copy> ImmTy<'tcx, Tag> {
 }
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
-    /// Normalice `place.ptr` to a `Pointer` if this is a place and not a ZST.
+    /// Normalize `place.ptr` to a `Pointer` if this is a place and not a ZST.
     /// Can be helpful to avoid lots of `force_ptr` calls later, if this place is used a lot.
     #[inline]
     pub fn force_op_ptr(
@@ -532,7 +524,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 // * During ConstProp, with `TooGeneric` or since the `requried_consts` were not all
                 //   checked yet.
                 // * During CTFE, since promoteds in `const`/`static` initializer bodies can fail.
-                self.const_to_op(val, layout)?
+
+                self.mir_const_to_op(&val, layout)?
             }
         };
         trace!("{:?}: {:?}", mir_op, *op);
@@ -556,28 +549,45 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         val: &ty::Const<'tcx>,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
-        let tag_scalar = |scalar| -> InterpResult<'tcx, _> {
-            Ok(match scalar {
-                Scalar::Ptr(ptr) => Scalar::Ptr(self.global_base_pointer(ptr)?),
-                Scalar::Int(int) => Scalar::Int(int),
-            })
-        };
-        // Early-return cases.
-        let val_val = match val.val {
+        match val.val {
             ty::ConstKind::Param(_) | ty::ConstKind::Bound(..) => throw_inval!(TooGeneric),
             ty::ConstKind::Error(_) => throw_inval!(AlreadyReported(ErrorReported)),
-            ty::ConstKind::Unevaluated(def, substs, promoted) => {
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) => {
                 let instance = self.resolve(def, substs)?;
-                return Ok(self.eval_to_allocation(GlobalId { instance, promoted })?.into());
+                Ok(self.eval_to_allocation(GlobalId { instance, promoted })?.into())
             }
             ty::ConstKind::Infer(..) | ty::ConstKind::Placeholder(..) => {
                 span_bug!(self.cur_span(), "const_to_op: Unexpected ConstKind {:?}", val)
             }
-            ty::ConstKind::Value(val_val) => val_val,
-        };
+            ty::ConstKind::Value(val_val) => self.const_val_to_op(val_val, val.ty, layout),
+        }
+    }
+
+    crate fn mir_const_to_op(
+        &self,
+        val: &mir::ConstantKind<'tcx>,
+        layout: Option<TyAndLayout<'tcx>>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        match val {
+            mir::ConstantKind::Ty(ct) => self.const_to_op(ct, layout),
+            mir::ConstantKind::Val(val, ty) => self.const_val_to_op(*val, ty, layout),
+        }
+    }
+
+    crate fn const_val_to_op(
+        &self,
+        val_val: ConstValue<'tcx>,
+        ty: Ty<'tcx>,
+        layout: Option<TyAndLayout<'tcx>>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
         // Other cases need layout.
-        let layout =
-            from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(val.ty))?;
+        let tag_scalar = |scalar| -> InterpResult<'tcx, _> {
+            Ok(match scalar {
+                Scalar::Ptr(ptr) => Scalar::Ptr(self.global_base_pointer(ptr)?),
+                Scalar::Int(int) => Scalar::Int(int),
+            })
+        };
+        let layout = from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(ty))?;
         let op = match val_val {
             ConstValue::ByRef { alloc, offset } => {
                 let id = self.tcx.create_memory_alloc(alloc);
diff --git a/compiler/rustc_mir/src/interpret/place.rs b/compiler/rustc_mir/src/interpret/place.rs
index 392f739e84f..699b531f501 100644
--- a/compiler/rustc_mir/src/interpret/place.rs
+++ b/compiler/rustc_mir/src/interpret/place.rs
@@ -33,7 +33,7 @@ pub enum MemPlaceMeta<Tag = ()> {
     Poison,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(MemPlaceMeta, 24);
 
 impl<Tag> MemPlaceMeta<Tag> {
@@ -74,7 +74,7 @@ pub struct MemPlace<Tag = ()> {
     pub meta: MemPlaceMeta<Tag>,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(MemPlace, 56);
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, HashStable)]
@@ -87,7 +87,7 @@ pub enum Place<Tag = ()> {
     Local { frame: usize, local: mir::Local },
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(Place, 64);
 
 #[derive(Copy, Clone, Debug)]
@@ -96,7 +96,7 @@ pub struct PlaceTy<'tcx, Tag = ()> {
     pub layout: TyAndLayout<'tcx>,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(PlaceTy<'_>, 80);
 
 impl<'tcx, Tag> std::ops::Deref for PlaceTy<'tcx, Tag> {
@@ -114,7 +114,7 @@ pub struct MPlaceTy<'tcx, Tag = ()> {
     pub layout: TyAndLayout<'tcx>,
 }
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(MPlaceTy<'_>, 72);
 
 impl<'tcx, Tag> std::ops::Deref for MPlaceTy<'tcx, Tag> {
@@ -531,7 +531,7 @@ where
         base.offset(from_offset, meta, layout, self)
     }
 
-    pub(super) fn mplace_downcast(
+    pub(crate) fn mplace_downcast(
         &self,
         base: &MPlaceTy<'tcx, M::PointerTag>,
         variant: VariantIdx,
diff --git a/compiler/rustc_mir/src/interpret/step.rs b/compiler/rustc_mir/src/interpret/step.rs
index 64d7c8ef2c7..6084f67abd7 100644
--- a/compiler/rustc_mir/src/interpret/step.rs
+++ b/compiler/rustc_mir/src/interpret/step.rs
@@ -2,6 +2,7 @@
 //!
 //! The main entry point is the `step` method.
 
+use crate::interpret::OpTy;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::{InterpResult, Scalar};
 use rustc_target::abi::LayoutOf;
@@ -113,6 +114,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 M::retag(self, *kind, &dest)?;
             }
 
+            // Call CopyNonOverlapping
+            CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => {
+                let src = self.eval_operand(src, None)?;
+                let dst = self.eval_operand(dst, None)?;
+                let count = self.eval_operand(count, None)?;
+                self.copy(&src, &dst, &count, /* nonoverlapping */ true)?;
+            }
+
             // Statements we do not track.
             AscribeUserType(..) => {}
 
@@ -140,6 +149,37 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         Ok(())
     }
 
+    pub(crate) fn copy(
+        &mut self,
+        src: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>,
+        dst: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>,
+        count: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>,
+        nonoverlapping: bool,
+    ) -> InterpResult<'tcx> {
+        let count = self.read_scalar(&count)?.to_machine_usize(self)?;
+        let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap().ty)?;
+        let (size, align) = (layout.size, layout.align.abi);
+        let size = size.checked_mul(count, self).ok_or_else(|| {
+            err_ub_format!(
+                "overflow computing total size of `{}`",
+                if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
+            )
+        })?;
+
+        // Make sure we check both pointers for an access of the total size and aligment,
+        // *even if* the total size is 0.
+        let src =
+            self.memory.check_ptr_access(self.read_scalar(&src)?.check_init()?, size, align)?;
+
+        let dst =
+            self.memory.check_ptr_access(self.read_scalar(&dst)?.check_init()?, size, align)?;
+
+        if let (Some(src), Some(dst)) = (src, dst) {
+            self.memory.copy(src, dst, size, nonoverlapping)?;
+        }
+        Ok(())
+    }
+
     /// Evaluate an assignment statement.
     ///
     /// There is no separate `eval_rvalue` function. Instead, the code for handling each rvalue
@@ -165,7 +205,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 self.copy_op(&op, &dest)?;
             }
 
-            BinaryOp(bin_op, ref left, ref right) => {
+            BinaryOp(bin_op, box (ref left, ref right)) => {
                 let layout = binop_left_homogeneous(bin_op).then_some(dest.layout);
                 let left = self.read_immediate(&self.eval_operand(left, layout)?)?;
                 let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
@@ -173,7 +213,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 self.binop_ignore_overflow(bin_op, &left, &right, &dest)?;
             }
 
-            CheckedBinaryOp(bin_op, ref left, ref right) => {
+            CheckedBinaryOp(bin_op, box (ref left, ref right)) => {
                 // Due to the extra boolean in the result, we can never reuse the `dest.layout`.
                 let left = self.read_immediate(&self.eval_operand(left, None)?)?;
                 let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
diff --git a/compiler/rustc_mir/src/interpret/terminator.rs b/compiler/rustc_mir/src/interpret/terminator.rs
index 0807949a2d9..4aa1360d535 100644
--- a/compiler/rustc_mir/src/interpret/terminator.rs
+++ b/compiler/rustc_mir/src/interpret/terminator.rs
@@ -248,9 +248,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             };
             if normalize_abi(caller_abi) != normalize_abi(callee_abi) {
                 throw_ub_format!(
-                    "calling a function with ABI {:?} using caller ABI {:?}",
-                    callee_abi,
-                    caller_abi
+                    "calling a function with ABI {} using caller ABI {}",
+                    callee_abi.name(),
+                    caller_abi.name()
                 )
             }
         }
diff --git a/compiler/rustc_mir/src/interpret/validity.rs b/compiler/rustc_mir/src/interpret/validity.rs
index 2d2799f81e3..062ef7d8b4c 100644
--- a/compiler/rustc_mir/src/interpret/validity.rs
+++ b/compiler/rustc_mir/src/interpret/validity.rs
@@ -77,7 +77,7 @@ macro_rules! throw_validation_failure {
 ///
 macro_rules! try_validation {
     ($e:expr, $where:expr,
-     $( $( $p:pat )|+ => { $( $what_fmt:expr ),+ } $( expected { $( $expected_fmt:expr ),+ } )? ),+ $(,)?
+    $( $( $p:pat )|+ => { $( $what_fmt:expr ),+ } $( expected { $( $expected_fmt:expr ),+ } )? ),+ $(,)?
     ) => {{
         match $e {
             Ok(x) => x,
@@ -244,17 +244,20 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
             // generators and closures.
             ty::Closure(def_id, _) | ty::Generator(def_id, _, _) => {
                 let mut name = None;
-                if let Some(def_id) = def_id.as_local() {
-                    let tables = self.ecx.tcx.typeck(def_id);
-                    if let Some(upvars) = tables.closure_captures.get(&def_id.to_def_id()) {
+                // FIXME this should be more descriptive i.e. CapturePlace instead of CapturedVar
+                // https://github.com/rust-lang/project-rfc-2229/issues/46
+                if let Some(local_def_id) = def_id.as_local() {
+                    let tables = self.ecx.tcx.typeck(local_def_id);
+                    if let Some(captured_place) =
+                        tables.closure_min_captures_flattened(*def_id).nth(field)
+                    {
                         // Sometimes the index is beyond the number of upvars (seen
                         // for a generator).
-                        if let Some((&var_hir_id, _)) = upvars.get_index(field) {
-                            let node = self.ecx.tcx.hir().get(var_hir_id);
-                            if let hir::Node::Binding(pat) = node {
-                                if let hir::PatKind::Binding(_, _, ident, _) = pat.kind {
-                                    name = Some(ident.name);
-                                }
+                        let var_hir_id = captured_place.get_root_variable();
+                        let node = self.ecx.tcx.hir().get(var_hir_id);
+                        if let hir::Node::Binding(pat) = node {
+                            if let hir::PatKind::Binding(_, _, ident, _) = pat.kind {
+                                name = Some(ident.name);
                             }
                         }
                     }
diff --git a/compiler/rustc_mir/src/lib.rs b/compiler/rustc_mir/src/lib.rs
index 508510a81e1..b0db4f9e649 100644
--- a/compiler/rustc_mir/src/lib.rs
+++ b/compiler/rustc_mir/src/lib.rs
@@ -7,6 +7,7 @@ Rust MIR: a lowered representation of Rust.
 #![feature(nll)]
 #![feature(in_band_lifetimes)]
 #![feature(array_windows)]
+#![feature(assert_matches)]
 #![feature(bindings_after_at)]
 #![feature(bool_to_option)]
 #![feature(box_patterns)]
@@ -17,15 +18,17 @@ Rust MIR: a lowered representation of Rust.
 #![feature(decl_macro)]
 #![feature(exact_size_is_empty)]
 #![feature(exhaustive_patterns)]
+#![feature(iter_zip)]
 #![feature(never_type)]
+#![feature(map_try_insert)]
 #![feature(min_specialization)]
 #![feature(trusted_len)]
 #![feature(try_blocks)]
 #![feature(associated_type_defaults)]
 #![feature(stmt_expr_attributes)]
 #![feature(trait_alias)]
-#![feature(option_expect_none)]
-#![feature(or_patterns)]
+#![feature(option_get_or_insert_default)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(once_cell)]
 #![feature(control_flow_enum)]
 #![recursion_limit = "256"]
@@ -62,6 +65,10 @@ pub fn provide(providers: &mut Providers) {
         let (param_env, value) = param_env_and_value.into_parts();
         const_eval::destructure_const(tcx, param_env, value)
     };
+    providers.const_to_valtree = |tcx, param_env_and_value| {
+        let (param_env, raw) = param_env_and_value.into_parts();
+        const_eval::const_to_valtree(tcx, param_env, raw)
+    };
     providers.deref_const = |tcx, param_env_and_value| {
         let (param_env, value) = param_env_and_value.into_parts();
         const_eval::deref_const(tcx, param_env, value)
diff --git a/compiler/rustc_mir/src/monomorphize/collector.rs b/compiler/rustc_mir/src/monomorphize/collector.rs
index 20cb989196a..1fda71d74bb 100644
--- a/compiler/rustc_mir/src/monomorphize/collector.rs
+++ b/compiler/rustc_mir/src/monomorphize/collector.rs
@@ -59,11 +59,15 @@
 //!
 //! ### Discovering roots
 //!
-//! The roots of the mono item graph correspond to the non-generic
+//! The roots of the mono item graph correspond to the public non-generic
 //! syntactic items in the source code. We find them by walking the HIR of the
-//! crate, and whenever we hit upon a function, method, or static item, we
-//! create a mono item consisting of the items DefId and, since we only
-//! consider non-generic items, an empty type-substitution set.
+//! crate, and whenever we hit upon a public function, method, or static item,
+//! we create a mono item consisting of the items DefId and, since we only
+//! consider non-generic items, an empty type-substitution set. (In eager
+//! collection mode, during incremental compilation, all non-generic functions
+//! are considered as roots, as well as when the `-Clink-dead-code` option is
+//! specified. Functions marked `#[no_mangle]` and functions called by inlinable
+//! functions also always act as roots.)
 //!
 //! ### Finding neighbor nodes
 //! Given a mono item node, we can discover neighbors by inspecting its
@@ -184,7 +188,6 @@ use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId, LOCAL_CRATE};
 use rustc_hir::itemlikevisit::ItemLikeVisitor;
 use rustc_hir::lang_items::LangItem;
 use rustc_index::bit_set::GrowableBitSet;
-use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{AllocId, ConstValue};
 use rustc_middle::mir::interpret::{ErrorHandled, GlobalAlloc, Scalar};
 use rustc_middle::mir::mono::{InstantiationMode, MonoItem};
@@ -193,6 +196,7 @@ use rustc_middle::mir::{self, Local, Location};
 use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCast};
 use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts};
 use rustc_middle::ty::{self, GenericParamDefKind, Instance, Ty, TyCtxt, TypeFoldable};
+use rustc_middle::{middle::codegen_fn_attrs::CodegenFnAttrFlags, mir::visit::TyContext};
 use rustc_session::config::EntryFnType;
 use rustc_span::source_map::{dummy_spanned, respan, Span, Spanned, DUMMY_SP};
 use smallvec::SmallVec;
@@ -638,6 +642,35 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
         self.super_rvalue(rvalue, location);
     }
 
+    /// This does not walk the constant, as it has been handled entirely here and trying
+    /// to walk it would attempt to evaluate the `ty::Const` inside, which doesn't necessarily
+    /// work, as some constants cannot be represented in the type system.
+    fn visit_constant(&mut self, constant: &mir::Constant<'tcx>, location: Location) {
+        let literal = self.monomorphize(constant.literal);
+        let val = match literal {
+            mir::ConstantKind::Val(val, _) => val,
+            mir::ConstantKind::Ty(ct) => match ct.val {
+                ty::ConstKind::Value(val) => val,
+                ty::ConstKind::Unevaluated(ct) => {
+                    let param_env = ty::ParamEnv::reveal_all();
+                    match self.tcx.const_eval_resolve(param_env, ct, None) {
+                        // The `monomorphize` call should have evaluated that constant already.
+                        Ok(val) => val,
+                        Err(ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted) => return,
+                        Err(ErrorHandled::TooGeneric) => span_bug!(
+                            self.body.source_info(location).span,
+                            "collection encountered polymorphic constant: {:?}",
+                            literal
+                        ),
+                    }
+                }
+                _ => return,
+            },
+        };
+        collect_const_value(self.tcx, val, self.output);
+        self.visit_ty(literal.ty(), TyContext::Location(location));
+    }
+
     fn visit_const(&mut self, constant: &&'tcx ty::Const<'tcx>, location: Location) {
         debug!("visiting const {:?} @ {:?}", *constant, location);
 
@@ -646,9 +679,15 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
 
         match substituted_constant.val {
             ty::ConstKind::Value(val) => collect_const_value(self.tcx, val, self.output),
-            ty::ConstKind::Unevaluated(def, substs, promoted) => {
-                match self.tcx.const_eval_resolve(param_env, def, substs, promoted, None) {
-                    Ok(val) => collect_const_value(self.tcx, val, self.output),
+            ty::ConstKind::Unevaluated(unevaluated) => {
+                match self.tcx.const_eval_resolve(param_env, unevaluated, None) {
+                    // The `monomorphize` call should have evaluated that constant already.
+                    Ok(val) => span_bug!(
+                        self.body.source_info(location).span,
+                        "collection encountered the unevaluated constant {} which evaluated to {:?}",
+                        substituted_constant,
+                        val
+                    ),
                     Err(ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted) => {}
                     Err(ErrorHandled::TooGeneric) => span_bug!(
                         self.body.source_info(location).span,
@@ -684,7 +723,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
                 for op in operands {
                     match *op {
                         mir::InlineAsmOperand::SymFn { ref value } => {
-                            let fn_ty = self.monomorphize(value.literal.ty);
+                            let fn_ty = self.monomorphize(value.literal.ty());
                             visit_fn_use(self.tcx, fn_ty, false, source, &mut self.output);
                         }
                         mir::InlineAsmOperand::SymStatic { def_id } => {
@@ -1175,7 +1214,8 @@ fn create_mono_items_for_default_impls<'tcx>(
                     let substs =
                         InternalSubsts::for_item(tcx, method.def_id, |param, _| match param.kind {
                             GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
-                            GenericParamDefKind::Type { .. } | GenericParamDefKind::Const => {
+                            GenericParamDefKind::Type { .. }
+                            | GenericParamDefKind::Const { .. } => {
                                 trait_ref.substs[param.index as usize]
                             }
                         });
diff --git a/compiler/rustc_mir/src/monomorphize/mod.rs b/compiler/rustc_mir/src/monomorphize/mod.rs
index d2586f0f84d..9ca4b6687f1 100644
--- a/compiler/rustc_mir/src/monomorphize/mod.rs
+++ b/compiler/rustc_mir/src/monomorphize/mod.rs
@@ -8,14 +8,14 @@ pub mod collector;
 pub mod partitioning;
 pub mod polymorphize;
 
-pub fn custom_coerce_unsize_info<'tcx>(
+fn custom_coerce_unsize_info<'tcx>(
     tcx: TyCtxt<'tcx>,
     source_ty: Ty<'tcx>,
     target_ty: Ty<'tcx>,
 ) -> CustomCoerceUnsized {
     let def_id = tcx.require_lang_item(LangItem::CoerceUnsized, None);
 
-    let trait_ref = ty::Binder::bind(ty::TraitRef {
+    let trait_ref = ty::Binder::dummy(ty::TraitRef {
         def_id,
         substs: tcx.mk_substs_trait(source_ty, &[target_ty.into()]),
     });
diff --git a/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs b/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs
index b68a8104fba..dc2379fd92b 100644
--- a/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs
+++ b/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs
@@ -424,8 +424,33 @@ fn collect_and_partition_mono_items<'tcx>(
     (tcx.arena.alloc(mono_items), codegen_units)
 }
 
+fn codegened_and_inlined_items<'tcx>(tcx: TyCtxt<'tcx>, cnum: CrateNum) -> &'tcx DefIdSet {
+    let (items, cgus) = tcx.collect_and_partition_mono_items(cnum);
+    let mut visited = DefIdSet::default();
+    let mut result = items.clone();
+
+    for cgu in cgus {
+        for (item, _) in cgu.items() {
+            if let MonoItem::Fn(ref instance) = item {
+                let did = instance.def_id();
+                if !visited.insert(did) {
+                    continue;
+                }
+                for scope in &tcx.instance_mir(instance.def).source_scopes {
+                    if let Some((ref inlined, _)) = scope.inlined {
+                        result.insert(inlined.def_id());
+                    }
+                }
+            }
+        }
+    }
+
+    tcx.arena.alloc(result)
+}
+
 pub fn provide(providers: &mut Providers) {
     providers.collect_and_partition_mono_items = collect_and_partition_mono_items;
+    providers.codegened_and_inlined_items = codegened_and_inlined_items;
 
     providers.is_codegened_item = |tcx, def_id| {
         let (all_mono_items, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
diff --git a/compiler/rustc_mir/src/monomorphize/polymorphize.rs b/compiler/rustc_mir/src/monomorphize/polymorphize.rs
index 4ad71ab4913..30e758c7fdf 100644
--- a/compiler/rustc_mir/src/monomorphize/polymorphize.rs
+++ b/compiler/rustc_mir/src/monomorphize/polymorphize.rs
@@ -30,9 +30,8 @@ pub fn provide(providers: &mut Providers) {
 /// Determine which generic parameters are used by the function/method/closure represented by
 /// `def_id`. Returns a bitset where bits representing unused parameters are set (`is_empty`
 /// indicates all parameters are used).
+#[instrument(skip(tcx))]
 fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> {
-    debug!("unused_generic_params({:?})", def_id);
-
     if !tcx.sess.opts.debugging_opts.polymorphize {
         // If polymorphization disabled, then all parameters are used.
         return FiniteBitSet::new_empty();
@@ -46,7 +45,7 @@ fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> {
     }
 
     let generics = tcx.generics_of(def_id);
-    debug!("unused_generic_params: generics={:?}", generics);
+    debug!(?generics);
 
     // Exit early when there are no parameters to be unused.
     if generics.count() == 0 {
@@ -57,11 +56,11 @@ fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> {
     let context = tcx.hir().body_const_context(def_id.expect_local());
     match context {
         Some(ConstContext::ConstFn) | None if !tcx.is_mir_available(def_id) => {
-            debug!("unused_generic_params: (no mir available) def_id={:?}", def_id);
+            debug!("no mir available");
             return FiniteBitSet::new_empty();
         }
         Some(_) if !tcx.is_ctfe_mir_available(def_id) => {
-            debug!("unused_generic_params: (no ctfe mir available) def_id={:?}", def_id);
+            debug!("no ctfe mir available");
             return FiniteBitSet::new_empty();
         }
         _ => {}
@@ -72,9 +71,9 @@ fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> {
         generics.count().try_into().expect("more generic parameters than can fit into a `u32`");
     let mut unused_parameters = FiniteBitSet::<u32>::new_empty();
     unused_parameters.set_range(0..generics_count);
-    debug!("unused_generic_params: (start) unused_parameters={:?}", unused_parameters);
+    debug!(?unused_parameters, "(start)");
     mark_used_by_default_parameters(tcx, def_id, generics, &mut unused_parameters);
-    debug!("unused_generic_params: (after default) unused_parameters={:?}", unused_parameters);
+    debug!(?unused_parameters, "(after default)");
 
     // Visit MIR and accumululate used generic parameters.
     let body = match context {
@@ -85,10 +84,10 @@ fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> {
     };
     let mut vis = MarkUsedGenericParams { tcx, def_id, unused_parameters: &mut unused_parameters };
     vis.visit_body(body);
-    debug!("unused_generic_params: (after visitor) unused_parameters={:?}", unused_parameters);
+    debug!(?unused_parameters, "(after visitor)");
 
     mark_used_by_predicates(tcx, def_id, &mut unused_parameters);
-    debug!("unused_generic_params: (end) unused_parameters={:?}", unused_parameters);
+    debug!(?unused_parameters, "(end)");
 
     // Emit errors for debugging and testing if enabled.
     if !unused_parameters.is_empty() {
@@ -101,24 +100,55 @@ fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> {
 /// Some parameters are considered used-by-default, such as non-generic parameters and the dummy
 /// generic parameters from closures, this function marks them as used. `leaf_is_closure` should
 /// be `true` if the item that `unused_generic_params` was invoked on is a closure.
+#[instrument(skip(tcx, def_id, generics, unused_parameters))]
 fn mark_used_by_default_parameters<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
     generics: &'tcx ty::Generics,
     unused_parameters: &mut FiniteBitSet<u32>,
 ) {
-    if !tcx.is_trait(def_id) && (tcx.is_closure(def_id) || tcx.type_of(def_id).is_generator()) {
-        for param in &generics.params {
-            debug!("mark_used_by_default_parameters: (closure/gen) param={:?}", param);
-            unused_parameters.clear(param.index);
-        }
-    } else {
-        for param in &generics.params {
-            debug!("mark_used_by_default_parameters: (other) param={:?}", param);
-            if let ty::GenericParamDefKind::Lifetime = param.kind {
+    match tcx.def_kind(def_id) {
+        DefKind::Closure | DefKind::Generator => {
+            for param in &generics.params {
+                debug!(?param, "(closure/gen)");
                 unused_parameters.clear(param.index);
             }
         }
+        DefKind::Mod
+        | DefKind::Struct
+        | DefKind::Union
+        | DefKind::Enum
+        | DefKind::Variant
+        | DefKind::Trait
+        | DefKind::TyAlias
+        | DefKind::ForeignTy
+        | DefKind::TraitAlias
+        | DefKind::AssocTy
+        | DefKind::TyParam
+        | DefKind::Fn
+        | DefKind::Const
+        | DefKind::ConstParam
+        | DefKind::Static
+        | DefKind::Ctor(_, _)
+        | DefKind::AssocFn
+        | DefKind::AssocConst
+        | DefKind::Macro(_)
+        | DefKind::ExternCrate
+        | DefKind::Use
+        | DefKind::ForeignMod
+        | DefKind::AnonConst
+        | DefKind::OpaqueTy
+        | DefKind::Field
+        | DefKind::LifetimeParam
+        | DefKind::GlobalAsm
+        | DefKind::Impl => {
+            for param in &generics.params {
+                debug!(?param, "(other)");
+                if let ty::GenericParamDefKind::Lifetime = param.kind {
+                    unused_parameters.clear(param.index);
+                }
+            }
+        }
     }
 
     if let Some(parent) = generics.parent {
@@ -128,6 +158,7 @@ fn mark_used_by_default_parameters<'tcx>(
 
 /// Search the predicates on used generic parameters for any unused generic parameters, and mark
 /// those as used.
+#[instrument(skip(tcx, def_id))]
 fn mark_used_by_predicates<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
@@ -135,16 +166,12 @@ fn mark_used_by_predicates<'tcx>(
 ) {
     let def_id = tcx.closure_base_def_id(def_id);
     let predicates = tcx.explicit_predicates_of(def_id);
-    debug!("mark_used_by_predicates: predicates_of={:?}", predicates);
 
     let mut current_unused_parameters = FiniteBitSet::new_empty();
     // Run to a fixed point to support `where T: Trait<U>, U: Trait<V>`, starting with an empty
     // bit set so that this is skipped if all parameters are already used.
     while current_unused_parameters != *unused_parameters {
-        debug!(
-            "mark_used_by_predicates: current_unused_parameters={:?} = unused_parameters={:?}",
-            current_unused_parameters, unused_parameters
-        );
+        debug!(?current_unused_parameters, ?unused_parameters);
         current_unused_parameters = *unused_parameters;
 
         for (predicate, _) in predicates.predicates {
@@ -169,13 +196,13 @@ fn mark_used_by_predicates<'tcx>(
 
 /// Emit errors for the function annotated by `#[rustc_polymorphize_error]`, labelling each generic
 /// parameter which was unused.
+#[instrument(skip(tcx, generics))]
 fn emit_unused_generic_params_error<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
     generics: &'tcx ty::Generics,
     unused_parameters: &FiniteBitSet<u32>,
 ) {
-    debug!("emit_unused_generic_params_error: def_id={:?}", def_id);
     let base_def_id = tcx.closure_base_def_id(def_id);
     if !tcx
         .get_attrs(base_def_id)
@@ -185,7 +212,6 @@ fn emit_unused_generic_params_error<'tcx>(
         return;
     }
 
-    debug!("emit_unused_generic_params_error: unused_parameters={:?}", unused_parameters);
     let fn_span = match tcx.opt_item_name(def_id) {
         Some(ident) => ident.span,
         _ => tcx.def_span(def_id),
@@ -197,7 +223,7 @@ fn emit_unused_generic_params_error<'tcx>(
     while let Some(generics) = next_generics {
         for param in &generics.params {
             if unused_parameters.contains(param.index).unwrap_or(false) {
-                debug!("emit_unused_generic_params_error: param={:?}", param);
+                debug!(?param);
                 let def_span = tcx.def_span(param.def_id);
                 err.span_label(def_span, &format!("generic parameter `{}` is unused", param.name));
             }
@@ -219,25 +245,23 @@ struct MarkUsedGenericParams<'a, 'tcx> {
 impl<'a, 'tcx> MarkUsedGenericParams<'a, 'tcx> {
     /// Invoke `unused_generic_params` on a body contained within the current item (e.g.
     /// a closure, generator or constant).
+    #[instrument(skip(self, def_id, substs))]
     fn visit_child_body(&mut self, def_id: DefId, substs: SubstsRef<'tcx>) {
         let unused = self.tcx.unused_generic_params(def_id);
-        debug!(
-            "visit_child_body: unused_parameters={:?} unused={:?}",
-            self.unused_parameters, unused
-        );
+        debug!(?self.unused_parameters, ?unused);
         for (i, arg) in substs.iter().enumerate() {
             let i = i.try_into().unwrap();
             if !unused.contains(i).unwrap_or(false) {
                 arg.visit_with(self);
             }
         }
-        debug!("visit_child_body: unused_parameters={:?}", self.unused_parameters);
+        debug!(?self.unused_parameters);
     }
 }
 
 impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
+    #[instrument(skip(self, local))]
     fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) {
-        debug!("visit_local_decl: local_decl={:?}", local_decl);
         if local == Local::from_usize(1) {
             let def_kind = self.tcx.def_kind(self.def_id);
             if matches!(def_kind, DefKind::Closure | DefKind::Generator) {
@@ -245,7 +269,7 @@ impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
                 // happens because the first argument to the closure is a reference to itself and
                 // that will call `visit_substs`, resulting in each generic parameter captured being
                 // considered used by default.
-                debug!("visit_local_decl: skipping closure substs");
+                debug!("skipping closure substs");
                 return;
             }
         }
@@ -263,19 +287,19 @@ impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
 }
 
 impl<'a, 'tcx> TypeVisitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
+    #[instrument(skip(self))]
     fn visit_const(&mut self, c: &'tcx Const<'tcx>) -> ControlFlow<Self::BreakTy> {
-        debug!("visit_const: c={:?}", c);
         if !c.has_param_types_or_consts() {
             return ControlFlow::CONTINUE;
         }
 
         match c.val {
             ty::ConstKind::Param(param) => {
-                debug!("visit_const: param={:?}", param);
+                debug!(?param);
                 self.unused_parameters.clear(param.index);
                 ControlFlow::CONTINUE
             }
-            ty::ConstKind::Unevaluated(def, _, Some(p))
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs: _, promoted: Some(p)})
                 // Avoid considering `T` unused when constants are of the form:
                 //   `<Self as Foo<T>>::foo::promoted[p]`
                 if self.def_id == def.did && !self.tcx.generics_of(def.did).has_self =>
@@ -286,25 +310,25 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
                 self.visit_body(&promoted[p]);
                 ControlFlow::CONTINUE
             }
-            ty::ConstKind::Unevaluated(def, unevaluated_substs, None)
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted: None })
                 if self.tcx.def_kind(def.did) == DefKind::AnonConst =>
             {
-                self.visit_child_body(def.did, unevaluated_substs);
+                self.visit_child_body(def.did, substs);
                 ControlFlow::CONTINUE
             }
             _ => c.super_visit_with(self),
         }
     }
 
+    #[instrument(skip(self))]
     fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
-        debug!("visit_ty: ty={:?}", ty);
         if !ty.has_param_types_or_consts() {
             return ControlFlow::CONTINUE;
         }
 
         match *ty.kind() {
             ty::Closure(def_id, substs) | ty::Generator(def_id, substs, ..) => {
-                debug!("visit_ty: def_id={:?}", def_id);
+                debug!(?def_id);
                 // Avoid cycle errors with generators.
                 if def_id == self.def_id {
                     return ControlFlow::CONTINUE;
@@ -316,7 +340,7 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
                 ControlFlow::CONTINUE
             }
             ty::Param(param) => {
-                debug!("visit_ty: param={:?}", param);
+                debug!(?param);
                 self.unused_parameters.clear(param.index);
                 ControlFlow::CONTINUE
             }
@@ -333,8 +357,8 @@ struct HasUsedGenericParams<'a> {
 impl<'a, 'tcx> TypeVisitor<'tcx> for HasUsedGenericParams<'a> {
     type BreakTy = ();
 
+    #[instrument(skip(self))]
     fn visit_const(&mut self, c: &'tcx Const<'tcx>) -> ControlFlow<Self::BreakTy> {
-        debug!("visit_const: c={:?}", c);
         if !c.has_param_types_or_consts() {
             return ControlFlow::CONTINUE;
         }
@@ -351,8 +375,8 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for HasUsedGenericParams<'a> {
         }
     }
 
+    #[instrument(skip(self))]
     fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
-        debug!("visit_ty: ty={:?}", ty);
         if !ty.has_param_types_or_consts() {
             return ControlFlow::CONTINUE;
         }
diff --git a/compiler/rustc_mir/src/shim.rs b/compiler/rustc_mir/src/shim.rs
index b246ec5c814..796d024771d 100644
--- a/compiler/rustc_mir/src/shim.rs
+++ b/compiler/rustc_mir/src/shim.rs
@@ -421,7 +421,7 @@ impl CloneShimBuilder<'tcx> {
         let func = Operand::Constant(box Constant {
             span: self.span,
             user_ty: None,
-            literal: ty::Const::zero_sized(tcx, func_ty),
+            literal: ty::Const::zero_sized(tcx, func_ty).into(),
         });
 
         let ref_loc = self.make_place(
@@ -463,7 +463,7 @@ impl CloneShimBuilder<'tcx> {
         let cond = self.make_place(Mutability::Mut, tcx.types.bool);
         let compute_cond = self.make_statement(StatementKind::Assign(box (
             cond,
-            Rvalue::BinaryOp(BinOp::Ne, Operand::Copy(end), Operand::Copy(beg)),
+            Rvalue::BinaryOp(BinOp::Ne, box (Operand::Copy(end), Operand::Copy(beg))),
         )));
 
         // `if end != beg { goto loop_body; } else { goto loop_end; }`
@@ -478,7 +478,7 @@ impl CloneShimBuilder<'tcx> {
         box Constant {
             span: self.span,
             user_ty: None,
-            literal: ty::Const::from_usize(self.tcx, value),
+            literal: ty::Const::from_usize(self.tcx, value).into(),
         }
     }
 
@@ -509,7 +509,7 @@ impl CloneShimBuilder<'tcx> {
                 Rvalue::Use(Operand::Constant(box Constant {
                     span: self.span,
                     user_ty: None,
-                    literal: len,
+                    literal: len.into(),
                 })),
             ))),
         ];
@@ -536,8 +536,7 @@ impl CloneShimBuilder<'tcx> {
             Place::from(beg),
             Rvalue::BinaryOp(
                 BinOp::Add,
-                Operand::Copy(Place::from(beg)),
-                Operand::Constant(self.make_usize(1)),
+                box (Operand::Copy(Place::from(beg)), Operand::Constant(self.make_usize(1))),
             ),
         )))];
         self.block(statements, TerminatorKind::Goto { target: BasicBlock::new(1) }, false);
@@ -590,8 +589,7 @@ impl CloneShimBuilder<'tcx> {
             Place::from(beg),
             Rvalue::BinaryOp(
                 BinOp::Add,
-                Operand::Copy(Place::from(beg)),
-                Operand::Constant(self.make_usize(1)),
+                box (Operand::Copy(Place::from(beg)), Operand::Constant(self.make_usize(1))),
             ),
         )));
         self.block(vec![statement], TerminatorKind::Goto { target: BasicBlock::new(6) }, true);
@@ -770,7 +768,7 @@ fn build_call_shim<'tcx>(
                 Operand::Constant(box Constant {
                     span,
                     user_ty: None,
-                    literal: ty::Const::zero_sized(tcx, ty),
+                    literal: ty::Const::zero_sized(tcx, ty).into(),
                 }),
                 rcvr.into_iter().collect::<Vec<_>>(),
             )
diff --git a/compiler/rustc_mir/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_mir/src/transform/check_consts/post_drop_elaboration.rs
index 1a2d932ba19..057092b8ef5 100644
--- a/compiler/rustc_mir/src/transform/check_consts/post_drop_elaboration.rs
+++ b/compiler/rustc_mir/src/transform/check_consts/post_drop_elaboration.rs
@@ -79,7 +79,9 @@ impl Visitor<'tcx> for CheckLiveDrops<'mir, 'tcx> {
             mir::TerminatorKind::Drop { place: dropped_place, .. } => {
                 let dropped_ty = dropped_place.ty(self.body, self.tcx).ty;
                 if !NeedsDrop::in_any_value_of_ty(self.ccx, dropped_ty) {
-                    return;
+                    bug!(
+                        "Drop elaboration left behind a Drop for a type that does not need dropping"
+                    );
                 }
 
                 if dropped_place.is_indirect() {
@@ -87,6 +89,10 @@ impl Visitor<'tcx> for CheckLiveDrops<'mir, 'tcx> {
                     return;
                 }
 
+                // Drop elaboration is not precise enough to accept code like
+                // `src/test/ui/consts/control-flow/drop-pass.rs`; e.g., when an `Option<Vec<T>>` is
+                // initialized with `None` and never changed, it still emits drop glue.
+                // Hence we additionally check the qualifs here to allow more code to pass.
                 if self.qualifs.needs_drop(self.ccx, dropped_place.local, location) {
                     // Use the span where the dropped local was declared for the error.
                     let span = self.body.local_decls[dropped_place.local].source_info.span;
diff --git a/compiler/rustc_mir/src/transform/check_consts/qualifs.rs b/compiler/rustc_mir/src/transform/check_consts/qualifs.rs
index 0ce1980f10a..ac8c748ea85 100644
--- a/compiler/rustc_mir/src/transform/check_consts/qualifs.rs
+++ b/compiler/rustc_mir/src/transform/check_consts/qualifs.rs
@@ -29,11 +29,11 @@ pub fn in_any_value_of_ty(
 /// Normally, we would determine what qualifications apply to each type and error when an illegal
 /// operation is performed on such a type. However, this was found to be too imprecise, especially
 /// in the presence of `enum`s. If only a single variant of an enum has a certain qualification, we
-/// needn't reject code unless it actually constructs and operates on the qualifed variant.
+/// needn't reject code unless it actually constructs and operates on the qualified variant.
 ///
 /// To accomplish this, const-checking and promotion use a value-based analysis (as opposed to a
 /// type-based one). Qualifications propagate structurally across variables: If a local (or a
-/// projection of a local) is assigned a qualifed value, that local itself becomes qualifed.
+/// projection of a local) is assigned a qualified value, that local itself becomes qualified.
 pub trait Qualif {
     /// The name of the file used to debug the dataflow analysis that computes this qualif.
     const ANALYSIS_NAME: &'static str;
@@ -168,7 +168,7 @@ where
         | Rvalue::UnaryOp(_, operand)
         | Rvalue::Cast(_, operand, _) => in_operand::<Q, _>(cx, in_local, operand),
 
-        Rvalue::BinaryOp(_, lhs, rhs) | Rvalue::CheckedBinaryOp(_, lhs, rhs) => {
+        Rvalue::BinaryOp(_, box (lhs, rhs)) | Rvalue::CheckedBinaryOp(_, box (lhs, rhs)) => {
             in_operand::<Q, _>(cx, in_local, lhs) || in_operand::<Q, _>(cx, in_local, rhs)
         }
 
@@ -246,25 +246,27 @@ where
     };
 
     // Check the qualifs of the value of `const` items.
-    if let ty::ConstKind::Unevaluated(def, _, promoted) = constant.literal.val {
-        assert!(promoted.is_none());
-        // Don't peek inside trait associated constants.
-        if cx.tcx.trait_of_item(def.did).is_none() {
-            let qualifs = if let Some((did, param_did)) = def.as_const_arg() {
-                cx.tcx.at(constant.span).mir_const_qualif_const_arg((did, param_did))
-            } else {
-                cx.tcx.at(constant.span).mir_const_qualif(def.did)
-            };
-
-            if !Q::in_qualifs(&qualifs) {
-                return false;
-            }
+    if let Some(ct) = constant.literal.const_for_ty() {
+        if let ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs: _, promoted }) = ct.val {
+            assert!(promoted.is_none());
+            // Don't peek inside trait associated constants.
+            if cx.tcx.trait_of_item(def.did).is_none() {
+                let qualifs = if let Some((did, param_did)) = def.as_const_arg() {
+                    cx.tcx.at(constant.span).mir_const_qualif_const_arg((did, param_did))
+                } else {
+                    cx.tcx.at(constant.span).mir_const_qualif(def.did)
+                };
+
+                if !Q::in_qualifs(&qualifs) {
+                    return false;
+                }
 
-            // Just in case the type is more specific than
-            // the definition, e.g., impl associated const
-            // with type parameters, take it into account.
+                // Just in case the type is more specific than
+                // the definition, e.g., impl associated const
+                // with type parameters, take it into account.
+            }
         }
     }
     // Otherwise use the qualifs of the type.
-    Q::in_any_value_of_ty(cx, constant.literal.ty)
+    Q::in_any_value_of_ty(cx, constant.literal.ty())
 }
diff --git a/compiler/rustc_mir/src/transform/check_consts/validation.rs b/compiler/rustc_mir/src/transform/check_consts/validation.rs
index baa27e9499f..ce5e41d3e7c 100644
--- a/compiler/rustc_mir/src/transform/check_consts/validation.rs
+++ b/compiler/rustc_mir/src/transform/check_consts/validation.rs
@@ -684,8 +684,8 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> {
                 }
             }
 
-            Rvalue::BinaryOp(op, ref lhs, ref rhs)
-            | Rvalue::CheckedBinaryOp(op, ref lhs, ref rhs) => {
+            Rvalue::BinaryOp(op, box (ref lhs, ref rhs))
+            | Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => {
                 let lhs_ty = lhs.ty(self.body, self.tcx);
                 let rhs_ty = rhs.ty(self.body, self.tcx);
 
@@ -808,6 +808,7 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> {
             | StatementKind::Retag { .. }
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::Nop => {}
         }
     }
@@ -849,9 +850,12 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> {
                     let obligation = Obligation::new(
                         ObligationCause::dummy(),
                         param_env,
-                        Binder::bind(TraitPredicate {
-                            trait_ref: TraitRef::from_method(tcx, trait_id, substs),
-                        }),
+                        Binder::bind(
+                            TraitPredicate {
+                                trait_ref: TraitRef::from_method(tcx, trait_id, substs),
+                            },
+                            tcx,
+                        ),
                     );
 
                     let implsrc = tcx.infer_ctxt().enter(|infcx| {
diff --git a/compiler/rustc_mir/src/transform/check_packed_ref.rs b/compiler/rustc_mir/src/transform/check_packed_ref.rs
index ee88daa83e7..13b7221046b 100644
--- a/compiler/rustc_mir/src/transform/check_packed_ref.rs
+++ b/compiler/rustc_mir/src/transform/check_packed_ref.rs
@@ -1,11 +1,18 @@
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::*;
+use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::lint::builtin::UNALIGNED_REFERENCES;
+use rustc_span::symbol::sym;
 
 use crate::transform::MirPass;
 use crate::util;
 
+pub(crate) fn provide(providers: &mut Providers) {
+    *providers = Providers { unsafe_derive_on_repr_packed, ..*providers };
+}
+
 pub struct CheckPackedRef;
 
 impl<'tcx> MirPass<'tcx> for CheckPackedRef {
@@ -24,6 +31,41 @@ struct PackedRefChecker<'a, 'tcx> {
     source_info: SourceInfo,
 }
 
+fn unsafe_derive_on_repr_packed(tcx: TyCtxt<'_>, def_id: LocalDefId) {
+    let lint_hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+
+    tcx.struct_span_lint_hir(UNALIGNED_REFERENCES, lint_hir_id, tcx.def_span(def_id), |lint| {
+        // FIXME: when we make this a hard error, this should have its
+        // own error code.
+        let message = if tcx.generics_of(def_id).own_requires_monomorphization() {
+            "`#[derive]` can't be used on a `#[repr(packed)]` struct with \
+             type or const parameters (error E0133)"
+                .to_string()
+        } else {
+            "`#[derive]` can't be used on a `#[repr(packed)]` struct that \
+             does not derive Copy (error E0133)"
+                .to_string()
+        };
+        lint.build(&message).emit()
+    });
+}
+
+fn builtin_derive_def_id(tcx: TyCtxt<'_>, def_id: DefId) -> Option<DefId> {
+    debug!("builtin_derive_def_id({:?})", def_id);
+    if let Some(impl_def_id) = tcx.impl_of_method(def_id) {
+        if tcx.has_attr(impl_def_id, sym::automatically_derived) {
+            debug!("builtin_derive_def_id({:?}) - is {:?}", def_id, impl_def_id);
+            Some(impl_def_id)
+        } else {
+            debug!("builtin_derive_def_id({:?}) - not automatically derived", def_id);
+            None
+        }
+    } else {
+        debug!("builtin_derive_def_id({:?}) - not a method", def_id);
+        None
+    }
+}
+
 impl<'a, 'tcx> Visitor<'tcx> for PackedRefChecker<'a, 'tcx> {
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
         // Make sure we know where in the MIR we are.
@@ -40,26 +82,33 @@ impl<'a, 'tcx> Visitor<'tcx> for PackedRefChecker<'a, 'tcx> {
     fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) {
         if context.is_borrow() {
             if util::is_disaligned(self.tcx, self.body, self.param_env, *place) {
-                let source_info = self.source_info;
-                let lint_root = self.body.source_scopes[source_info.scope]
-                    .local_data
-                    .as_ref()
-                    .assert_crate_local()
-                    .lint_root;
-                self.tcx.struct_span_lint_hir(
-                    UNALIGNED_REFERENCES,
-                    lint_root,
-                    source_info.span,
-                    |lint| {
-                        lint.build("reference to packed field is unaligned")
-                            .note(
-                                "fields of packed structs are not properly aligned, and creating \
-                                a misaligned reference is undefined behavior (even if that \
-                                reference is never dereferenced)",
-                            )
-                            .emit()
-                    },
-                );
+                let def_id = self.body.source.instance.def_id();
+                if let Some(impl_def_id) = builtin_derive_def_id(self.tcx, def_id) {
+                    // If a method is defined in the local crate,
+                    // the impl containing that method should also be.
+                    self.tcx.ensure().unsafe_derive_on_repr_packed(impl_def_id.expect_local());
+                } else {
+                    let source_info = self.source_info;
+                    let lint_root = self.body.source_scopes[source_info.scope]
+                        .local_data
+                        .as_ref()
+                        .assert_crate_local()
+                        .lint_root;
+                    self.tcx.struct_span_lint_hir(
+                        UNALIGNED_REFERENCES,
+                        lint_root,
+                        source_info.span,
+                        |lint| {
+                            lint.build("reference to packed field is unaligned")
+                                .note(
+                                    "fields of packed structs are not properly aligned, and creating \
+                                    a misaligned reference is undefined behavior (even if that \
+                                    reference is never dereferenced)",
+                                )
+                                .emit()
+                        },
+                    );
+                }
             }
         }
     }
diff --git a/compiler/rustc_mir/src/transform/check_unsafety.rs b/compiler/rustc_mir/src/transform/check_unsafety.rs
index f0472758dfb..09da9b2e4d6 100644
--- a/compiler/rustc_mir/src/transform/check_unsafety.rs
+++ b/compiler/rustc_mir/src/transform/check_unsafety.rs
@@ -10,14 +10,12 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::cast::CastTy;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_session::lint::builtin::{SAFE_PACKED_BORROWS, UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE};
+use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE};
 use rustc_session::lint::Level;
-use rustc_span::symbol::sym;
 
 use std::ops::Bound;
 
 use crate::const_eval::is_min_const_fn;
-use crate::util;
 
 pub struct UnsafetyChecker<'a, 'tcx> {
     body: &'a Body<'tcx>,
@@ -123,6 +121,7 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> {
                 UnsafetyViolationKind::General,
                 UnsafetyViolationDetails::UseOfInlineAssembly,
             ),
+            StatementKind::CopyNonOverlapping(..) => unreachable!(),
         }
         self.super_statement(statement, location);
     }
@@ -181,18 +180,6 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> {
             self.check_mut_borrowing_layout_constrained_field(*place, context.is_mutating_use());
         }
 
-        // Check for borrows to packed fields.
-        // `is_disaligned` already traverses the place to consider all projections after the last
-        // `Deref`, so this only needs to be called once at the top level.
-        if context.is_borrow() {
-            if util::is_disaligned(self.tcx, self.body, self.param_env, *place) {
-                self.require_unsafe(
-                    UnsafetyViolationKind::BorrowPacked,
-                    UnsafetyViolationDetails::BorrowOfPackedField,
-                );
-            }
-        }
-
         // Some checks below need the extra metainfo of the local declaration.
         let decl = &self.body.local_decls[place.local];
 
@@ -316,47 +303,32 @@ impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> {
             // `unsafe` blocks are required in safe code
             Safety::Safe => {
                 for violation in violations {
-                    let mut violation = *violation;
                     match violation.kind {
                         UnsafetyViolationKind::GeneralAndConstFn
                         | UnsafetyViolationKind::General => {}
-                        UnsafetyViolationKind::BorrowPacked => {
-                            if self.min_const_fn {
-                                // const fns don't need to be backwards compatible and can
-                                // emit these violations as a hard error instead of a backwards
-                                // compat lint
-                                violation.kind = UnsafetyViolationKind::General;
-                            }
-                        }
-                        UnsafetyViolationKind::UnsafeFn
-                        | UnsafetyViolationKind::UnsafeFnBorrowPacked => {
+                        UnsafetyViolationKind::UnsafeFn => {
                             bug!("`UnsafetyViolationKind::UnsafeFn` in an `Safe` context")
                         }
                     }
-                    if !self.violations.contains(&violation) {
-                        self.violations.push(violation)
+                    if !self.violations.contains(violation) {
+                        self.violations.push(*violation)
                     }
                 }
                 false
             }
             // With the RFC 2585, no longer allow `unsafe` operations in `unsafe fn`s
-            Safety::FnUnsafe if self.tcx.features().unsafe_block_in_unsafe_fn => {
+            Safety::FnUnsafe => {
                 for violation in violations {
                     let mut violation = *violation;
 
-                    if violation.kind == UnsafetyViolationKind::BorrowPacked {
-                        violation.kind = UnsafetyViolationKind::UnsafeFnBorrowPacked;
-                    } else {
-                        violation.kind = UnsafetyViolationKind::UnsafeFn;
-                    }
+                    violation.kind = UnsafetyViolationKind::UnsafeFn;
                     if !self.violations.contains(&violation) {
                         self.violations.push(violation)
                     }
                 }
                 false
             }
-            // `unsafe` function bodies allow unsafe without additional unsafe blocks (before RFC 2585)
-            Safety::BuiltinUnsafe | Safety::FnUnsafe => true,
+            Safety::BuiltinUnsafe => true,
             Safety::ExplicitUnsafe(hir_id) => {
                 // mark unsafe block as used if there are any unsafe operations inside
                 if !violations.is_empty() {
@@ -369,8 +341,7 @@ impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> {
                             // these unsafe things are stable in const fn
                             UnsafetyViolationKind::GeneralAndConstFn => {}
                             // these things are forbidden in const fns
-                            UnsafetyViolationKind::General
-                            | UnsafetyViolationKind::BorrowPacked => {
+                            UnsafetyViolationKind::General => {
                                 let mut violation = *violation;
                                 // const fns don't need to be backwards compatible and can
                                 // emit these violations as a hard error instead of a backwards
@@ -380,8 +351,7 @@ impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> {
                                     self.violations.push(violation)
                                 }
                             }
-                            UnsafetyViolationKind::UnsafeFn
-                            | UnsafetyViolationKind::UnsafeFnBorrowPacked => bug!(
+                            UnsafetyViolationKind::UnsafeFn => bug!(
                                 "`UnsafetyViolationKind::UnsafeFn` in an `ExplicitUnsafe` context"
                             ),
                         }
@@ -464,7 +434,6 @@ pub(crate) fn provide(providers: &mut Providers) {
                 ty::WithOptConstParam { did, const_param_did: Some(param_did) },
             )
         },
-        unsafe_derive_on_repr_packed,
         ..*providers
     };
 }
@@ -544,25 +513,6 @@ fn unsafety_check_result<'tcx>(
     })
 }
 
-fn unsafe_derive_on_repr_packed(tcx: TyCtxt<'_>, def_id: LocalDefId) {
-    let lint_hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-
-    tcx.struct_span_lint_hir(SAFE_PACKED_BORROWS, lint_hir_id, tcx.def_span(def_id), |lint| {
-        // FIXME: when we make this a hard error, this should have its
-        // own error code.
-        let message = if tcx.generics_of(def_id).own_requires_monomorphization() {
-            "`#[derive]` can't be used on a `#[repr(packed)]` struct with \
-             type or const parameters (error E0133)"
-                .to_string()
-        } else {
-            "`#[derive]` can't be used on a `#[repr(packed)]` struct that \
-             does not derive Copy (error E0133)"
-                .to_string()
-        };
-        lint.build(&message).emit()
-    });
-}
-
 /// Returns the `HirId` for an enclosing scope that is also `unsafe`.
 fn is_enclosed(
     tcx: TyCtxt<'_>,
@@ -609,22 +559,6 @@ fn report_unused_unsafe(tcx: TyCtxt<'_>, used_unsafe: &FxHashSet<hir::HirId>, id
     });
 }
 
-fn builtin_derive_def_id(tcx: TyCtxt<'_>, def_id: DefId) -> Option<DefId> {
-    debug!("builtin_derive_def_id({:?})", def_id);
-    if let Some(impl_def_id) = tcx.impl_of_method(def_id) {
-        if tcx.has_attr(impl_def_id, sym::automatically_derived) {
-            debug!("builtin_derive_def_id({:?}) - is {:?}", def_id, impl_def_id);
-            Some(impl_def_id)
-        } else {
-            debug!("builtin_derive_def_id({:?}) - not automatically derived", def_id);
-            None
-        }
-    } else {
-        debug!("builtin_derive_def_id({:?}) - not a method", def_id);
-        None
-    }
-}
-
 pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) {
     debug!("check_unsafety({:?})", def_id);
 
@@ -657,27 +591,6 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) {
                 .note(note)
                 .emit();
             }
-            UnsafetyViolationKind::BorrowPacked => {
-                if let Some(impl_def_id) = builtin_derive_def_id(tcx, def_id.to_def_id()) {
-                    // If a method is defined in the local crate,
-                    // the impl containing that method should also be.
-                    tcx.ensure().unsafe_derive_on_repr_packed(impl_def_id.expect_local());
-                } else {
-                    tcx.struct_span_lint_hir(
-                        SAFE_PACKED_BORROWS,
-                        lint_root,
-                        source_info.span,
-                        |lint| {
-                            lint.build(&format!(
-                                "{} is unsafe and requires unsafe{} block (error E0133)",
-                                description, unsafe_fn_msg,
-                            ))
-                            .note(note)
-                            .emit()
-                        },
-                    )
-                }
-            }
             UnsafetyViolationKind::UnsafeFn => tcx.struct_span_lint_hir(
                 UNSAFE_OP_IN_UNSAFE_FN,
                 lint_root,
@@ -692,35 +605,6 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) {
                     .emit();
                 },
             ),
-            UnsafetyViolationKind::UnsafeFnBorrowPacked => {
-                // When `unsafe_op_in_unsafe_fn` is disallowed, the behavior of safe and unsafe functions
-                // should be the same in terms of warnings and errors. Therefore, with `#[warn(safe_packed_borrows)]`,
-                // a safe packed borrow should emit a warning *but not an error* in an unsafe function,
-                // just like in a safe function, even if `unsafe_op_in_unsafe_fn` is `deny`.
-                //
-                // Also, `#[warn(unsafe_op_in_unsafe_fn)]` can't cause any new errors. Therefore, with
-                // `#[deny(safe_packed_borrows)]` and `#[warn(unsafe_op_in_unsafe_fn)]`, a packed borrow
-                // should only issue a warning for the sake of backwards compatibility.
-                //
-                // The solution those 2 expectations is to always take the minimum of both lints.
-                // This prevent any new errors (unless both lints are explicitly set to `deny`).
-                let lint = if tcx.lint_level_at_node(SAFE_PACKED_BORROWS, lint_root).0
-                    <= tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, lint_root).0
-                {
-                    SAFE_PACKED_BORROWS
-                } else {
-                    UNSAFE_OP_IN_UNSAFE_FN
-                };
-                tcx.struct_span_lint_hir(&lint, lint_root, source_info.span, |lint| {
-                    lint.build(&format!(
-                        "{} is unsafe and requires unsafe block (error E0133)",
-                        description,
-                    ))
-                    .span_label(source_info.span, description)
-                    .note(note)
-                    .emit();
-                })
-            }
         }
     }
 
diff --git a/compiler/rustc_mir/src/transform/const_goto.rs b/compiler/rustc_mir/src/transform/const_goto.rs
index 3eb2e757644..b5c8b4bebc3 100644
--- a/compiler/rustc_mir/src/transform/const_goto.rs
+++ b/compiler/rustc_mir/src/transform/const_goto.rs
@@ -28,7 +28,7 @@ pub struct ConstGoto;
 
 impl<'tcx> MirPass<'tcx> for ConstGoto {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        if tcx.sess.opts.debugging_opts.mir_opt_level < 3 {
+        if tcx.sess.mir_opt_level() < 4 {
             return;
         }
         trace!("Running ConstGoto on {:?}", body.source);
diff --git a/compiler/rustc_mir/src/transform/const_prop.rs b/compiler/rustc_mir/src/transform/const_prop.rs
index 2a0e5e2c9c5..7706316c965 100644
--- a/compiler/rustc_mir/src/transform/const_prop.rs
+++ b/compiler/rustc_mir/src/transform/const_prop.rs
@@ -13,9 +13,9 @@ use rustc_middle::mir::visit::{
     MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor,
 };
 use rustc_middle::mir::{
-    AssertKind, BasicBlock, BinOp, Body, ClearCrossCrate, Constant, Local, LocalDecl, LocalKind,
-    Location, Operand, Place, Rvalue, SourceInfo, SourceScope, SourceScopeData, Statement,
-    StatementKind, Terminator, TerminatorKind, UnOp, RETURN_PLACE,
+    AssertKind, BasicBlock, BinOp, Body, ClearCrossCrate, Constant, ConstantKind, Local, LocalDecl,
+    LocalKind, Location, Operand, Place, Rvalue, SourceInfo, SourceScope, SourceScopeData,
+    Statement, StatementKind, Terminator, TerminatorKind, UnOp, RETURN_PLACE,
 };
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutError, TyAndLayout};
 use rustc_middle::ty::subst::{InternalSubsts, Subst};
@@ -482,18 +482,25 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             return None;
         }
 
-        match self.ecx.const_to_op(c.literal, None) {
+        match self.ecx.mir_const_to_op(&c.literal, None) {
             Ok(op) => Some(op),
             Err(error) => {
                 let tcx = self.ecx.tcx.at(c.span);
                 let err = ConstEvalErr::new(&self.ecx, error, Some(c.span));
                 if let Some(lint_root) = self.lint_root(source_info) {
-                    let lint_only = match c.literal.val {
-                        // Promoteds must lint and not error as the user didn't ask for them
-                        ConstKind::Unevaluated(_, _, Some(_)) => true,
-                        // Out of backwards compatibility we cannot report hard errors in unused
-                        // generic functions using associated constants of the generic parameters.
-                        _ => c.literal.needs_subst(),
+                    let lint_only = match c.literal {
+                        ConstantKind::Ty(ct) => match ct.val {
+                            // Promoteds must lint and not error as the user didn't ask for them
+                            ConstKind::Unevaluated(ty::Unevaluated {
+                                def: _,
+                                substs: _,
+                                promoted: Some(_),
+                            }) => true,
+                            // Out of backwards compatibility we cannot report hard errors in unused
+                            // generic functions using associated constants of the generic parameters.
+                            _ => c.literal.needs_subst(),
+                        },
+                        ConstantKind::Val(_, ty) => ty.needs_subst(),
                     };
                     if lint_only {
                         // Out of backwards compatibility we cannot report hard errors in unused
@@ -676,11 +683,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                 trace!("checking UnaryOp(op = {:?}, arg = {:?})", op, arg);
                 self.check_unary_op(*op, arg, source_info)?;
             }
-            Rvalue::BinaryOp(op, left, right) => {
+            Rvalue::BinaryOp(op, box (left, right)) => {
                 trace!("checking BinaryOp(op = {:?}, left = {:?}, right = {:?})", op, left, right);
                 self.check_binary_op(*op, left, right, source_info)?;
             }
-            Rvalue::CheckedBinaryOp(op, left, right) => {
+            Rvalue::CheckedBinaryOp(op, box (left, right)) => {
                 trace!(
                     "checking CheckedBinaryOp(op = {:?}, left = {:?}, right = {:?})",
                     op,
@@ -725,7 +732,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             return None;
         }
 
-        if self.tcx.sess.opts.debugging_opts.mir_opt_level >= 3 {
+        if self.tcx.sess.mir_opt_level() >= 4 {
             self.eval_rvalue_with_identities(rvalue, place)
         } else {
             self.use_ecx(|this| this.ecx.eval_rvalue_into_place(rvalue, place))
@@ -740,7 +747,8 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     ) -> Option<()> {
         self.use_ecx(|this| {
             match rvalue {
-                Rvalue::BinaryOp(op, left, right) | Rvalue::CheckedBinaryOp(op, left, right) => {
+                Rvalue::BinaryOp(op, box (left, right))
+                | Rvalue::CheckedBinaryOp(op, box (left, right)) => {
                     let l = this.ecx.eval_operand(left, None);
                     let r = this.ecx.eval_operand(right, None);
 
@@ -772,7 +780,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                         }
                         BinOp::Mul => {
                             if const_arg.layout.ty.is_integral() && arg_value == 0 {
-                                if let Rvalue::CheckedBinaryOp(_, _, _) = rvalue {
+                                if let Rvalue::CheckedBinaryOp(_, _) = rvalue {
                                     let val = Immediate::ScalarPair(
                                         const_arg.to_scalar()?.into(),
                                         Scalar::from_bool(false).into(),
@@ -802,7 +810,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         Operand::Constant(Box::new(Constant {
             span,
             user_ty: None,
-            literal: ty::Const::from_scalar(self.tcx, scalar, ty),
+            literal: ty::Const::from_scalar(self.tcx, scalar, ty).into(),
         }))
     }
 
@@ -813,9 +821,12 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         source_info: SourceInfo,
     ) {
         if let Rvalue::Use(Operand::Constant(c)) = rval {
-            if !matches!(c.literal.val, ConstKind::Unevaluated(..)) {
-                trace!("skipping replace of Rvalue::Use({:?} because it is already a const", c);
-                return;
+            match c.literal {
+                ConstantKind::Ty(c) if matches!(c.val, ConstKind::Unevaluated(..)) => {}
+                _ => {
+                    trace!("skipping replace of Rvalue::Use({:?} because it is already a const", c);
+                    return;
+                }
             }
         }
 
@@ -882,13 +893,17 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                                 *rval = Rvalue::Use(Operand::Constant(Box::new(Constant {
                                     span: source_info.span,
                                     user_ty: None,
-                                    literal: self.ecx.tcx.mk_const(ty::Const {
-                                        ty,
-                                        val: ty::ConstKind::Value(ConstValue::ByRef {
-                                            alloc,
-                                            offset: Size::ZERO,
-                                        }),
-                                    }),
+                                    literal: self
+                                        .ecx
+                                        .tcx
+                                        .mk_const(ty::Const {
+                                            ty,
+                                            val: ty::ConstKind::Value(ConstValue::ByRef {
+                                                alloc,
+                                                offset: Size::ZERO,
+                                            }),
+                                        })
+                                        .into(),
                                 })));
                             }
                         }
@@ -903,7 +918,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
 
     /// Returns `true` if and only if this `op` should be const-propagated into.
     fn should_const_prop(&mut self, op: &OpTy<'tcx>) -> bool {
-        let mir_opt_level = self.tcx.sess.opts.debugging_opts.mir_opt_level;
+        let mir_opt_level = self.tcx.sess.mir_opt_level();
 
         if mir_opt_level == 0 {
             return false;
@@ -1071,9 +1086,9 @@ impl<'mir, 'tcx> MutVisitor<'tcx> for ConstPropagator<'mir, 'tcx> {
     fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) {
         self.super_operand(operand, location);
 
-        // Only const prop copies and moves on `mir_opt_level=2` as doing so
+        // Only const prop copies and moves on `mir_opt_level=3` as doing so
         // currently slightly increases compile time in some cases.
-        if self.tcx.sess.opts.debugging_opts.mir_opt_level >= 2 {
+        if self.tcx.sess.mir_opt_level() >= 3 {
             self.propagate_operand(operand)
         }
     }
@@ -1253,7 +1268,7 @@ impl<'mir, 'tcx> MutVisitor<'tcx> for ConstPropagator<'mir, 'tcx> {
             TerminatorKind::SwitchInt { ref mut discr, .. } => {
                 // FIXME: This is currently redundant with `visit_operand`, but sadly
                 // always visiting operands currently causes a perf regression in LLVM codegen, so
-                // `visit_operand` currently only runs for propagates places for `mir_opt_level=3`.
+                // `visit_operand` currently only runs for propagates places for `mir_opt_level=4`.
                 self.propagate_operand(discr)
             }
             // None of these have Operands to const-propagate.
@@ -1272,7 +1287,7 @@ impl<'mir, 'tcx> MutVisitor<'tcx> for ConstPropagator<'mir, 'tcx> {
             // Every argument in our function calls have already been propagated in `visit_operand`.
             //
             // NOTE: because LLVM codegen gives slight performance regressions with it, so this is
-            // gated on `mir_opt_level=2`.
+            // gated on `mir_opt_level=3`.
             TerminatorKind::Call { .. } => {}
         }
 
diff --git a/compiler/rustc_mir/src/transform/coverage/debug.rs b/compiler/rustc_mir/src/transform/coverage/debug.rs
index 2cd0dc6b1f2..48361483099 100644
--- a/compiler/rustc_mir/src/transform/coverage/debug.rs
+++ b/compiler/rustc_mir/src/transform/coverage/debug.rs
@@ -121,6 +121,7 @@ use rustc_middle::mir::coverage::*;
 use rustc_middle::mir::{self, BasicBlock, TerminatorKind};
 use rustc_middle::ty::TyCtxt;
 
+use std::iter;
 use std::lazy::SyncOnceCell;
 
 pub const NESTED_INDENT: &str = "    ";
@@ -285,10 +286,8 @@ impl DebugCounters {
                 ),
             };
             counters
-                .insert(id, DebugCounter::new(counter_kind.clone(), some_block_label))
-                .expect_none(
-                    "attempt to add the same counter_kind to DebugCounters more than once",
-                );
+                .try_insert(id, DebugCounter::new(counter_kind.clone(), some_block_label))
+                .expect("attempt to add the same counter_kind to DebugCounters more than once");
         }
     }
 
@@ -479,9 +478,9 @@ impl GraphvizData {
         counter_kind: &CoverageKind,
     ) {
         if let Some(edge_to_counter) = self.some_edge_to_counter.as_mut() {
-            edge_to_counter.insert((from_bcb, to_bb), counter_kind.clone()).expect_none(
-                "invalid attempt to insert more than one edge counter for the same edge",
-            );
+            edge_to_counter
+                .try_insert((from_bcb, to_bb), counter_kind.clone())
+                .expect("invalid attempt to insert more than one edge counter for the same edge");
         }
     }
 
@@ -705,9 +704,7 @@ pub(super) fn dump_coverage_graphviz(
         let edge_counters = from_terminator
             .successors()
             .map(|&successor_bb| graphviz_data.get_edge_counter(from_bcb, successor_bb));
-        edge_labels
-            .iter()
-            .zip(edge_counters)
+        iter::zip(&edge_labels, edge_counters)
             .map(|(label, some_counter)| {
                 if let Some(counter) = some_counter {
                     format!("{}\n{}", label, debug_counters.format_counter(counter))
@@ -819,7 +816,7 @@ fn bcb_to_string_sections(
     sections
 }
 
-/// Returns a simple string representation of a `TerminatorKind` variant, indenpendent of any
+/// Returns a simple string representation of a `TerminatorKind` variant, independent of any
 /// values it might hold.
 pub(super) fn term_type(kind: &TerminatorKind<'tcx>) -> &'static str {
     match kind {
diff --git a/compiler/rustc_mir/src/transform/coverage/graph.rs b/compiler/rustc_mir/src/transform/coverage/graph.rs
index e58b915f126..6f5fa858e25 100644
--- a/compiler/rustc_mir/src/transform/coverage/graph.rs
+++ b/compiler/rustc_mir/src/transform/coverage/graph.rs
@@ -392,10 +392,8 @@ impl BasicCoverageBlockData {
             }
         }
         let operand = counter_kind.as_operand_id();
-        if let Some(replaced) = self
-            .edge_from_bcbs
-            .get_or_insert_with(FxHashMap::default)
-            .insert(from_bcb, counter_kind)
+        if let Some(replaced) =
+            self.edge_from_bcbs.get_or_insert_default().insert(from_bcb, counter_kind)
         {
             Error::from_string(format!(
                 "attempt to set an edge counter more than once; from_bcb: \
diff --git a/compiler/rustc_mir/src/transform/coverage/mod.rs b/compiler/rustc_mir/src/transform/coverage/mod.rs
index 93133e9b7f0..60757178bec 100644
--- a/compiler/rustc_mir/src/transform/coverage/mod.rs
+++ b/compiler/rustc_mir/src/transform/coverage/mod.rs
@@ -111,7 +111,8 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
         let body_span = hir_body.value.span;
         let source_file = source_map.lookup_source_file(body_span.lo());
         let fn_sig_span = match some_fn_sig.filter(|fn_sig| {
-            Lrc::ptr_eq(&source_file, &source_map.lookup_source_file(fn_sig.span.hi()))
+            fn_sig.span.ctxt() == body_span.ctxt()
+                && Lrc::ptr_eq(&source_file, &source_map.lookup_source_file(fn_sig.span.hi()))
         }) {
             Some(fn_sig) => fn_sig.span.with_hi(body_span.lo()),
             None => body_span.shrink_to_lo(),
diff --git a/compiler/rustc_mir/src/transform/coverage/query.rs b/compiler/rustc_mir/src/transform/coverage/query.rs
index 4b455a6a1ba..2ba9d1bdc0c 100644
--- a/compiler/rustc_mir/src/transform/coverage/query.rs
+++ b/compiler/rustc_mir/src/transform/coverage/query.rs
@@ -1,16 +1,14 @@
 use super::*;
 
 use rustc_middle::mir::coverage::*;
-use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{self, Coverage, CoverageInfo, Location};
+use rustc_middle::mir::{self, Body, Coverage, CoverageInfo};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::def_id::DefId;
 
-/// The `query` provider for `CoverageInfo`, requested by `codegen_coverage()` (to inject each
-/// counter) and `FunctionCoverage::new()` (to extract the coverage map metadata from the MIR).
+/// A `query` provider for retrieving coverage information injected into MIR.
 pub(crate) fn provide(providers: &mut Providers) {
-    providers.coverageinfo = |tcx, def_id| coverageinfo_from_mir(tcx, def_id);
+    providers.coverageinfo = |tcx, def_id| coverageinfo(tcx, def_id);
     providers.covered_file_name = |tcx, def_id| covered_file_name(tcx, def_id);
     providers.covered_code_regions = |tcx, def_id| covered_code_regions(tcx, def_id);
 }
@@ -85,10 +83,21 @@ impl CoverageVisitor {
             }
         }
     }
-}
 
-impl Visitor<'_> for CoverageVisitor {
-    fn visit_coverage(&mut self, coverage: &Coverage, _location: Location) {
+    fn visit_body(&mut self, body: &Body<'_>) {
+        for bb_data in body.basic_blocks().iter() {
+            for statement in bb_data.statements.iter() {
+                if let StatementKind::Coverage(box ref coverage) = statement.kind {
+                    if is_inlined(body, statement) {
+                        continue;
+                    }
+                    self.visit_coverage(coverage);
+                }
+            }
+        }
+    }
+
+    fn visit_coverage(&mut self, coverage: &Coverage) {
         if self.add_missing_operands {
             match coverage.kind {
                 CoverageKind::Expression { lhs, rhs, .. } => {
@@ -111,7 +120,7 @@ impl Visitor<'_> for CoverageVisitor {
     }
 }
 
-fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> CoverageInfo {
+fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> CoverageInfo {
     let mir_body = mir_body(tcx, def_id);
 
     let mut coverage_visitor = CoverageVisitor {
@@ -129,35 +138,36 @@ fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> CoverageInfo
 }
 
 fn covered_file_name<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<Symbol> {
-    for bb_data in mir_body(tcx, def_id).basic_blocks().iter() {
-        for statement in bb_data.statements.iter() {
-            if let StatementKind::Coverage(box ref coverage) = statement.kind {
-                if let Some(code_region) = coverage.code_region.as_ref() {
-                    return Some(code_region.file_name);
+    if tcx.is_mir_available(def_id) {
+        let body = mir_body(tcx, def_id);
+        for bb_data in body.basic_blocks().iter() {
+            for statement in bb_data.statements.iter() {
+                if let StatementKind::Coverage(box ref coverage) = statement.kind {
+                    if let Some(code_region) = coverage.code_region.as_ref() {
+                        if is_inlined(body, statement) {
+                            continue;
+                        }
+                        return Some(code_region.file_name);
+                    }
                 }
             }
         }
     }
-    None
-}
-
-/// This function ensures we obtain the correct MIR for the given item irrespective of
-/// whether that means const mir or runtime mir. For `const fn` this opts for runtime
-/// mir.
-fn mir_body<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx mir::Body<'tcx> {
-    let id = ty::WithOptConstParam::unknown(def_id);
-    let def = ty::InstanceDef::Item(id);
-    tcx.instance_mir(def)
+    return None;
 }
 
 fn covered_code_regions<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Vec<&'tcx CodeRegion> {
-    mir_body(tcx, def_id)
-        .basic_blocks()
+    let body = mir_body(tcx, def_id);
+    body.basic_blocks()
         .iter()
         .map(|data| {
             data.statements.iter().filter_map(|statement| match statement.kind {
                 StatementKind::Coverage(box ref coverage) => {
-                    coverage.code_region.as_ref() // may be None
+                    if is_inlined(body, statement) {
+                        None
+                    } else {
+                        coverage.code_region.as_ref() // may be None
+                    }
                 }
                 _ => None,
             })
@@ -165,3 +175,17 @@ fn covered_code_regions<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Vec<&'tcx Cod
         .flatten()
         .collect()
 }
+
+fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool {
+    let scope_data = &body.source_scopes[statement.source_info.scope];
+    scope_data.inlined.is_some() || scope_data.inlined_parent_scope.is_some()
+}
+
+/// This function ensures we obtain the correct MIR for the given item irrespective of
+/// whether that means const mir or runtime mir. For `const fn` this opts for runtime
+/// mir.
+fn mir_body<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx mir::Body<'tcx> {
+    let id = ty::WithOptConstParam::unknown(def_id);
+    let def = ty::InstanceDef::Item(id);
+    tcx.instance_mir(def)
+}
diff --git a/compiler/rustc_mir/src/transform/coverage/spans.rs b/compiler/rustc_mir/src/transform/coverage/spans.rs
index fd3e782f6df..249f5e835cd 100644
--- a/compiler/rustc_mir/src/transform/coverage/spans.rs
+++ b/compiler/rustc_mir/src/transform/coverage/spans.rs
@@ -11,7 +11,7 @@ use rustc_middle::mir::{
 use rustc_middle::ty::TyCtxt;
 
 use rustc_span::source_map::original_sp;
-use rustc_span::{BytePos, Span, SyntaxContext};
+use rustc_span::{BytePos, Span};
 
 use std::cmp::Ordering;
 
@@ -240,7 +240,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
     /// to be).
     pub(super) fn generate_coverage_spans(
         mir_body: &'a mir::Body<'tcx>,
-        fn_sig_span: Span,
+        fn_sig_span: Span, // Ensured to be same SourceFile and SyntaxContext as `body_span`
         body_span: Span,
         basic_coverage_blocks: &'a CoverageGraph,
     ) -> Vec<CoverageSpan> {
@@ -683,10 +683,11 @@ pub(super) fn filtered_statement_span(
         // and `_1` is the `Place` for `somenum`.
         //
         // If and when the Issue is resolved, remove this special case match pattern:
-        StatementKind::FakeRead(cause, _) if cause == FakeReadCause::ForGuardBinding => None,
+        StatementKind::FakeRead(box (cause, _)) if cause == FakeReadCause::ForGuardBinding => None,
 
         // Retain spans from all other statements
-        StatementKind::FakeRead(_, _) // Not including `ForGuardBinding`
+        StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding`
+        | StatementKind::CopyNonOverlapping(..)
         | StatementKind::Assign(_)
         | StatementKind::SetDiscriminant { .. }
         | StatementKind::LlvmInlineAsm(_)
@@ -732,6 +733,6 @@ pub(super) fn filtered_terminator_span(
 
 #[inline]
 fn function_source_span(span: Span, body_span: Span) -> Span {
-    let span = original_sp(span, body_span).with_ctxt(SyntaxContext::root());
+    let span = original_sp(span, body_span).with_ctxt(body_span.ctxt());
     if body_span.contains(span) { span } else { body_span }
 }
diff --git a/compiler/rustc_mir/src/transform/coverage/tests.rs b/compiler/rustc_mir/src/transform/coverage/tests.rs
index 7a9bfaad883..dee112443d3 100644
--- a/compiler/rustc_mir/src/transform/coverage/tests.rs
+++ b/compiler/rustc_mir/src/transform/coverage/tests.rs
@@ -17,7 +17,7 @@
 //! Also note, some basic features of `Span` also rely on the `Span`s own "session globals", which
 //! are unrelated to the `TyCtxt` global. Without initializing the `Span` session globals, some
 //! basic, coverage-specific features would be impossible to test, but thankfully initializing these
-//! globals is comparitively simpler. The easiest way is to wrap the test in a closure argument
+//! globals is comparatively simpler. The easiest way is to wrap the test in a closure argument
 //! to: `rustc_span::with_default_session_globals(|| { test_here(); })`.
 
 use super::counters;
diff --git a/compiler/rustc_mir/src/transform/deduplicate_blocks.rs b/compiler/rustc_mir/src/transform/deduplicate_blocks.rs
index 5f09159e91b..c41e71e09a4 100644
--- a/compiler/rustc_mir/src/transform/deduplicate_blocks.rs
+++ b/compiler/rustc_mir/src/transform/deduplicate_blocks.rs
@@ -1,7 +1,7 @@
 //! This pass finds basic blocks that are completely equal,
 //! and replaces all uses with just one of them.
 
-use std::{collections::hash_map::Entry, hash::Hash, hash::Hasher};
+use std::{collections::hash_map::Entry, hash::Hash, hash::Hasher, iter};
 
 use crate::transform::MirPass;
 
@@ -16,7 +16,7 @@ pub struct DeduplicateBlocks;
 
 impl<'tcx> MirPass<'tcx> for DeduplicateBlocks {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        if tcx.sess.opts.debugging_opts.mir_opt_level < 3 {
+        if tcx.sess.mir_opt_level() < 4 {
             return;
         }
         debug!("Running DeduplicateBlocks on `{:?}`", body.source);
@@ -86,7 +86,7 @@ fn find_duplicates<'a, 'tcx>(body: &'a Body<'tcx>) -> FxHashMap<BasicBlock, Basi
                 // The basic block was already in the hashmap, which means we have a duplicate
                 let value = *occupied.get();
                 debug!("Inserting {:?} -> {:?}", bb, value);
-                duplicates.insert(bb, value).expect_none("key was already inserted");
+                duplicates.try_insert(bb, value).expect("key was already inserted");
             }
             Entry::Vacant(vacant) => {
                 vacant.insert(bb);
@@ -115,11 +115,7 @@ impl<'tcx, 'a> PartialEq for BasicBlockHashable<'tcx, 'a> {
     fn eq(&self, other: &Self) -> bool {
         self.basic_block_data.statements.len() == other.basic_block_data.statements.len()
             && &self.basic_block_data.terminator().kind == &other.basic_block_data.terminator().kind
-            && self
-                .basic_block_data
-                .statements
-                .iter()
-                .zip(&other.basic_block_data.statements)
+            && iter::zip(&self.basic_block_data.statements, &other.basic_block_data.statements)
                 .all(|(x, y)| statement_eq(&x.kind, &y.kind))
     }
 }
diff --git a/compiler/rustc_mir/src/transform/dest_prop.rs b/compiler/rustc_mir/src/transform/dest_prop.rs
index 46de5dba6e0..29df86ca6cd 100644
--- a/compiler/rustc_mir/src/transform/dest_prop.rs
+++ b/compiler/rustc_mir/src/transform/dest_prop.rs
@@ -127,9 +127,14 @@ pub struct DestinationPropagation;
 
 impl<'tcx> MirPass<'tcx> for DestinationPropagation {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        // Only run at mir-opt-level=2 or higher for now (we don't fix up debuginfo and remove
+        //  FIXME(#79191, #82678)
+        if !tcx.sess.opts.debugging_opts.unsound_mir_opts {
+            return;
+        }
+
+        // Only run at mir-opt-level=3 or higher for now (we don't fix up debuginfo and remove
         // storage statements at the moment).
-        if tcx.sess.opts.debugging_opts.mir_opt_level <= 1 {
+        if tcx.sess.mir_opt_level() < 3 {
             return;
         }
 
@@ -582,6 +587,7 @@ impl Conflicts<'a> {
             | StatementKind::FakeRead(..)
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::Nop => {}
         }
     }
@@ -714,9 +720,6 @@ impl Conflicts<'a> {
                                 }
                             }
                         }
-                        InlineAsmOperand::Const { value } => {
-                            assert!(value.place().is_none());
-                        }
                         InlineAsmOperand::InOut {
                             reg: _,
                             late: _,
@@ -725,6 +728,7 @@ impl Conflicts<'a> {
                         }
                         | InlineAsmOperand::In { reg: _, value: _ }
                         | InlineAsmOperand::Out { reg: _, late: _, place: None }
+                        | InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
                         | InlineAsmOperand::SymStatic { def_id: _ } => {}
                     }
diff --git a/compiler/rustc_mir/src/transform/early_otherwise_branch.rs b/compiler/rustc_mir/src/transform/early_otherwise_branch.rs
index b16a99d7f0d..f7ea9faec47 100644
--- a/compiler/rustc_mir/src/transform/early_otherwise_branch.rs
+++ b/compiler/rustc_mir/src/transform/early_otherwise_branch.rs
@@ -26,7 +26,12 @@ pub struct EarlyOtherwiseBranch;
 
 impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        if tcx.sess.opts.debugging_opts.mir_opt_level < 2 {
+        //  FIXME(#78496)
+        if !tcx.sess.opts.debugging_opts.unsound_mir_opts {
+            return;
+        }
+
+        if tcx.sess.mir_opt_level() < 3 {
             return;
         }
         trace!("running EarlyOtherwiseBranch on {:?}", body.source);
@@ -91,8 +96,10 @@ impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch {
                 opt_to_apply.infos[0].first_switch_info.discr_used_in_switch;
             let not_equal_rvalue = Rvalue::BinaryOp(
                 not_equal,
-                Operand::Copy(Place::from(second_discriminant_temp)),
-                Operand::Copy(first_descriminant_place),
+                box (
+                    Operand::Copy(Place::from(second_discriminant_temp)),
+                    Operand::Copy(first_descriminant_place),
+                ),
             );
             patch.add_statement(
                 end_of_block_location,
diff --git a/compiler/rustc_mir/src/transform/elaborate_drops.rs b/compiler/rustc_mir/src/transform/elaborate_drops.rs
index 3d435f6d0e7..c0fcfb620ff 100644
--- a/compiler/rustc_mir/src/transform/elaborate_drops.rs
+++ b/compiler/rustc_mir/src/transform/elaborate_drops.rs
@@ -471,7 +471,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
         Rvalue::Use(Operand::Constant(Box::new(Constant {
             span,
             user_ty: None,
-            literal: ty::Const::from_bool(self.tcx, val),
+            literal: ty::Const::from_bool(self.tcx, val).into(),
         })))
     }
 
diff --git a/compiler/rustc_mir/src/transform/generator.rs b/compiler/rustc_mir/src/transform/generator.rs
index 7a1f3d44a5e..003003a8abb 100644
--- a/compiler/rustc_mir/src/transform/generator.rs
+++ b/compiler/rustc_mir/src/transform/generator.rs
@@ -751,9 +751,10 @@ fn sanitize_witness<'tcx>(
             span_bug!(
                 body.span,
                 "Broken MIR: generator contains type {} in MIR, \
-                       but typeck only knows about {}",
-                decl.ty,
-                witness,
+                       but typeck only knows about {} and {:?}",
+                decl_ty,
+                allowed,
+                allowed_upvars
             );
         }
     }
@@ -989,7 +990,7 @@ fn insert_panic_block<'tcx>(
         cond: Operand::Constant(box Constant {
             span: body.span,
             user_ty: None,
-            literal: ty::Const::from_bool(tcx, false),
+            literal: ty::Const::from_bool(tcx, false).into(),
         }),
         expected: true,
         msg: message,
@@ -1454,6 +1455,7 @@ impl Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
             | StatementKind::Retag(..)
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::Nop => {}
         }
     }
diff --git a/compiler/rustc_mir/src/transform/inline.rs b/compiler/rustc_mir/src/transform/inline.rs
index 0a899273efe..b6f80763bc8 100644
--- a/compiler/rustc_mir/src/transform/inline.rs
+++ b/compiler/rustc_mir/src/transform/inline.rs
@@ -39,20 +39,11 @@ struct CallSite<'tcx> {
 
 /// Returns true if MIR inlining is enabled in the current compilation session.
 crate fn is_enabled(tcx: TyCtxt<'_>) -> bool {
-    if tcx.sess.opts.debugging_opts.instrument_coverage {
-        // Since `Inline` happens after `InstrumentCoverage`, the function-specific coverage
-        // counters can be invalidated, such as by merging coverage counter statements from
-        // a pre-inlined function into a different function. This kind of change is invalid,
-        // so inlining must be skipped. Note: This check is performed here so inlining can
-        // be disabled without preventing other optimizations (regardless of `mir_opt_level`).
-        return false;
-    }
-
     if let Some(enabled) = tcx.sess.opts.debugging_opts.inline_mir {
         return enabled;
     }
 
-    tcx.sess.opts.debugging_opts.mir_opt_level >= 2
+    tcx.sess.mir_opt_level() >= 3
 }
 
 impl<'tcx> MirPass<'tcx> for Inline {
@@ -416,7 +407,7 @@ impl Inliner<'tcx> {
 
                 TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
                     if let ty::FnDef(def_id, substs) =
-                        *callsite.callee.subst_mir(self.tcx, &f.literal.ty).kind()
+                        *callsite.callee.subst_mir(self.tcx, &f.literal.ty()).kind()
                     {
                         let substs = self.tcx.normalize_erasing_regions(self.param_env, substs);
                         if let Ok(Some(instance)) =
@@ -637,8 +628,11 @@ impl Inliner<'tcx> {
                 // `required_consts`, here we may not only have `ConstKind::Unevaluated`
                 // because we are calling `subst_and_normalize_erasing_regions`.
                 caller_body.required_consts.extend(
-                    callee_body.required_consts.iter().copied().filter(|&constant| {
-                        matches!(constant.literal.val, ConstKind::Unevaluated(_, _, _))
+                    callee_body.required_consts.iter().copied().filter(|&ct| {
+                        match ct.literal.const_for_ty() {
+                            Some(ct) => matches!(ct.val, ConstKind::Unevaluated(_)),
+                            None => true,
+                        }
                     }),
                 );
             }
diff --git a/compiler/rustc_mir/src/transform/instcombine.rs b/compiler/rustc_mir/src/transform/instcombine.rs
index 74dadb25725..7aaf0224164 100644
--- a/compiler/rustc_mir/src/transform/instcombine.rs
+++ b/compiler/rustc_mir/src/transform/instcombine.rs
@@ -44,7 +44,7 @@ impl<'tcx, 'a> InstCombineContext<'tcx, 'a> {
     /// Transform boolean comparisons into logical operations.
     fn combine_bool_cmp(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
         match rvalue {
-            Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), a, b) => {
+            Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), box (a, b)) => {
                 let new = match (op, self.try_eval_bool(a), self.try_eval_bool(b)) {
                     // Transform "Eq(a, true)" ==> "a"
                     (BinOp::Eq, _, Some(true)) => Some(a.clone()),
@@ -79,7 +79,7 @@ impl<'tcx, 'a> InstCombineContext<'tcx, 'a> {
 
     fn try_eval_bool(&self, a: &Operand<'_>) -> Option<bool> {
         let a = a.constant()?;
-        if a.literal.ty.is_bool() { a.literal.val.try_to_bool() } else { None }
+        if a.literal.ty().is_bool() { a.literal.try_to_bool() } else { None }
     }
 
     /// Transform "&(*a)" ==> "a".
@@ -110,12 +110,13 @@ impl<'tcx, 'a> InstCombineContext<'tcx, 'a> {
     fn combine_len(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
         if let Rvalue::Len(ref place) = *rvalue {
             let place_ty = place.ty(self.local_decls, self.tcx).ty;
-            if let ty::Array(_, len) = place_ty.kind() {
+            if let ty::Array(_, len) = *place_ty.kind() {
                 if !self.should_combine(source_info, rvalue) {
                     return;
                 }
 
-                let constant = Constant { span: source_info.span, literal: len, user_ty: None };
+                let constant =
+                    Constant { span: source_info.span, literal: len.into(), user_ty: None };
                 *rvalue = Rvalue::Use(Operand::Constant(box constant));
             }
         }
diff --git a/compiler/rustc_mir/src/transform/lower_intrinsics.rs b/compiler/rustc_mir/src/transform/lower_intrinsics.rs
index f5968532eb3..e6ee474285e 100644
--- a/compiler/rustc_mir/src/transform/lower_intrinsics.rs
+++ b/compiler/rustc_mir/src/transform/lower_intrinsics.rs
@@ -33,13 +33,34 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                                     Rvalue::Use(Operand::Constant(box Constant {
                                         span: terminator.source_info.span,
                                         user_ty: None,
-                                        literal: ty::Const::zero_sized(tcx, tcx.types.unit),
+                                        literal: ty::Const::zero_sized(tcx, tcx.types.unit).into(),
                                     })),
                                 )),
                             });
                             terminator.kind = TerminatorKind::Goto { target };
                         }
                     }
+                    sym::copy_nonoverlapping => {
+                        let target = destination.unwrap().1;
+                        let mut args = args.drain(..);
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::CopyNonOverlapping(
+                                box rustc_middle::mir::CopyNonOverlapping {
+                                    src: args.next().unwrap(),
+                                    dst: args.next().unwrap(),
+                                    count: args.next().unwrap(),
+                                },
+                            ),
+                        });
+                        assert_eq!(
+                            args.next(),
+                            None,
+                            "Extra argument for copy_non_overlapping intrinsic"
+                        );
+                        drop(args);
+                        terminator.kind = TerminatorKind::Goto { target };
+                    }
                     sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
                         if let Some((destination, target)) = *destination {
                             let lhs;
@@ -59,7 +80,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                                 source_info: terminator.source_info,
                                 kind: StatementKind::Assign(box (
                                     destination,
-                                    Rvalue::BinaryOp(bin_op, lhs, rhs),
+                                    Rvalue::BinaryOp(bin_op, box (lhs, rhs)),
                                 )),
                             });
                             terminator.kind = TerminatorKind::Goto { target };
diff --git a/compiler/rustc_mir/src/transform/match_branches.rs b/compiler/rustc_mir/src/transform/match_branches.rs
index 92b4ae397ae..f7a9835353e 100644
--- a/compiler/rustc_mir/src/transform/match_branches.rs
+++ b/compiler/rustc_mir/src/transform/match_branches.rs
@@ -1,6 +1,7 @@
 use crate::transform::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
+use std::iter;
 
 use super::simplify::simplify_cfg;
 
@@ -40,7 +41,7 @@ pub struct MatchBranchSimplification;
 
 impl<'tcx> MirPass<'tcx> for MatchBranchSimplification {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        if tcx.sess.opts.debugging_opts.mir_opt_level <= 1 {
+        if tcx.sess.mir_opt_level() < 3 {
             return;
         }
 
@@ -83,7 +84,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification {
             if first_stmts.len() != scnd_stmts.len() {
                 continue;
             }
-            for (f, s) in first_stmts.iter().zip(scnd_stmts.iter()) {
+            for (f, s) in iter::zip(first_stmts, scnd_stmts) {
                 match (&f.kind, &s.kind) {
                     // If two statements are exactly the same, we can optimize.
                     (f_s, s_s) if f_s == s_s => {}
@@ -93,8 +94,8 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification {
                         StatementKind::Assign(box (lhs_f, Rvalue::Use(Operand::Constant(f_c)))),
                         StatementKind::Assign(box (lhs_s, Rvalue::Use(Operand::Constant(s_c)))),
                     ) if lhs_f == lhs_s
-                        && f_c.literal.ty.is_bool()
-                        && s_c.literal.ty.is_bool()
+                        && f_c.literal.ty().is_bool()
+                        && s_c.literal.ty().is_bool()
                         && f_c.literal.try_eval_bool(tcx, param_env).is_some()
                         && s_c.literal.try_eval_bool(tcx, param_env).is_some() => {}
 
@@ -113,7 +114,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification {
             // and bb_idx has a different terminator from both of them.
             let (from, first, second) = bbs.pick3_mut(bb_idx, first, second);
 
-            let new_stmts = first.statements.iter().zip(second.statements.iter()).map(|(f, s)| {
+            let new_stmts = iter::zip(&first.statements, &second.statements).map(|(f, s)| {
                 match (&f.kind, &s.kind) {
                     (f_s, s_s) if f_s == s_s => (*f).clone(),
 
@@ -139,8 +140,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification {
                             let op = if f_b { BinOp::Eq } else { BinOp::Ne };
                             let rhs = Rvalue::BinaryOp(
                                 op,
-                                Operand::Copy(Place::from(discr_local)),
-                                const_cmp,
+                                box (Operand::Copy(Place::from(discr_local)), const_cmp),
                             );
                             Statement {
                                 source_info: f.source_info,
diff --git a/compiler/rustc_mir/src/transform/mod.rs b/compiler/rustc_mir/src/transform/mod.rs
index 9cb8abf75c4..5c49ee69edc 100644
--- a/compiler/rustc_mir/src/transform/mod.rs
+++ b/compiler/rustc_mir/src/transform/mod.rs
@@ -44,6 +44,7 @@ pub mod promote_consts;
 pub mod remove_noop_landing_pads;
 pub mod remove_storage_markers;
 pub mod remove_unneeded_drops;
+pub mod remove_zsts;
 pub mod required_consts;
 pub mod rustc_peek;
 pub mod simplify;
@@ -58,6 +59,7 @@ pub use rustc_middle::mir::MirSource;
 
 pub(crate) fn provide(providers: &mut Providers) {
     self::check_unsafety::provide(providers);
+    self::check_packed_ref::provide(providers);
     *providers = Providers {
         mir_keys,
         mir_const,
@@ -313,11 +315,8 @@ fn mir_promoted(
         &simplify::SimplifyCfg::new("promote-consts"),
     ];
 
-    let opt_coverage: &[&dyn MirPass<'tcx>] = if tcx.sess.opts.debugging_opts.instrument_coverage {
-        &[&coverage::InstrumentCoverage]
-    } else {
-        &[]
-    };
+    let opt_coverage: &[&dyn MirPass<'tcx>] =
+        if tcx.sess.instrument_coverage() { &[&coverage::InstrumentCoverage] } else { &[] };
 
     run_passes(tcx, &mut body, MirPhase::ConstPromotion, &[promote, opt_coverage]);
 
@@ -475,7 +474,7 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc
 }
 
 fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    let mir_opt_level = tcx.sess.opts.debugging_opts.mir_opt_level;
+    let mir_opt_level = tcx.sess.mir_opt_level();
 
     // Lowering generator control-flow and variables has to happen before we do anything else
     // to them. We run some optimizations before that, because they may be harder to do on the state
@@ -494,6 +493,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     // The main optimizations that we do on MIR.
     let optimizations: &[&dyn MirPass<'tcx>] = &[
         &remove_storage_markers::RemoveStorageMarkers,
+        &remove_zsts::RemoveZsts,
         &const_goto::ConstGoto,
         &remove_unneeded_drops::RemoveUnneededDrops,
         &match_branches::MatchBranchSimplification,
diff --git a/compiler/rustc_mir/src/transform/multiple_return_terminators.rs b/compiler/rustc_mir/src/transform/multiple_return_terminators.rs
index 617086622cc..4aaa0baa9f4 100644
--- a/compiler/rustc_mir/src/transform/multiple_return_terminators.rs
+++ b/compiler/rustc_mir/src/transform/multiple_return_terminators.rs
@@ -10,7 +10,7 @@ pub struct MultipleReturnTerminators;
 
 impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        if tcx.sess.opts.debugging_opts.mir_opt_level < 3 {
+        if tcx.sess.mir_opt_level() < 4 {
             return;
         }
 
diff --git a/compiler/rustc_mir/src/transform/nrvo.rs b/compiler/rustc_mir/src/transform/nrvo.rs
index ce02fb261df..445dc12909c 100644
--- a/compiler/rustc_mir/src/transform/nrvo.rs
+++ b/compiler/rustc_mir/src/transform/nrvo.rs
@@ -34,7 +34,7 @@ pub struct RenameReturnPlace;
 
 impl<'tcx> MirPass<'tcx> for RenameReturnPlace {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) {
-        if tcx.sess.opts.debugging_opts.mir_opt_level == 0 {
+        if tcx.sess.mir_opt_level() == 0 {
             return;
         }
 
diff --git a/compiler/rustc_mir/src/transform/promote_consts.rs b/compiler/rustc_mir/src/transform/promote_consts.rs
index 47d976281a4..1bbaf833c4f 100644
--- a/compiler/rustc_mir/src/transform/promote_consts.rs
+++ b/compiler/rustc_mir/src/transform/promote_consts.rs
@@ -108,9 +108,6 @@ pub enum Candidate {
     /// the attribute currently provides the semantic requirement that arguments
     /// must be constant.
     Argument { bb: BasicBlock, index: usize },
-
-    /// `const` operand in asm!.
-    InlineAsm { bb: BasicBlock, index: usize },
 }
 
 impl Candidate {
@@ -118,16 +115,14 @@ impl Candidate {
     fn forces_explicit_promotion(&self) -> bool {
         match self {
             Candidate::Ref(_) => false,
-            Candidate::Argument { .. } | Candidate::InlineAsm { .. } => true,
+            Candidate::Argument { .. } => true,
         }
     }
 
     fn source_info(&self, body: &Body<'_>) -> SourceInfo {
         match self {
             Candidate::Ref(location) => *body.source_info(*location),
-            Candidate::Argument { bb, .. } | Candidate::InlineAsm { bb, .. } => {
-                *body.source_info(body.terminator_loc(*bb))
-            }
+            Candidate::Argument { bb, .. } => *body.source_info(body.terminator_loc(*bb)),
         }
     }
 }
@@ -217,36 +212,25 @@ impl<'tcx> Visitor<'tcx> for Collector<'_, 'tcx> {
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
         self.super_terminator(terminator, location);
 
-        match terminator.kind {
-            TerminatorKind::Call { ref func, .. } => {
-                if let ty::FnDef(def_id, _) = *func.ty(self.ccx.body, self.ccx.tcx).kind() {
-                    let fn_sig = self.ccx.tcx.fn_sig(def_id);
-                    if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = fn_sig.abi() {
-                        let name = self.ccx.tcx.item_name(def_id);
-                        // FIXME(eddyb) use `#[rustc_args_required_const(2)]` for shuffles.
-                        if name.as_str().starts_with("simd_shuffle") {
-                            self.candidates
-                                .push(Candidate::Argument { bb: location.block, index: 2 });
-
-                            return; // Don't double count `simd_shuffle` candidates
-                        }
-                    }
+        if let TerminatorKind::Call { ref func, .. } = terminator.kind {
+            if let ty::FnDef(def_id, _) = *func.ty(self.ccx.body, self.ccx.tcx).kind() {
+                let fn_sig = self.ccx.tcx.fn_sig(def_id);
+                if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = fn_sig.abi() {
+                    let name = self.ccx.tcx.item_name(def_id);
+                    // FIXME(eddyb) use `#[rustc_args_required_const(2)]` for shuffles.
+                    if name.as_str().starts_with("simd_shuffle") {
+                        self.candidates.push(Candidate::Argument { bb: location.block, index: 2 });
 
-                    if let Some(constant_args) = args_required_const(self.ccx.tcx, def_id) {
-                        for index in constant_args {
-                            self.candidates.push(Candidate::Argument { bb: location.block, index });
-                        }
+                        return; // Don't double count `simd_shuffle` candidates
                     }
                 }
-            }
-            TerminatorKind::InlineAsm { ref operands, .. } => {
-                for (index, op) in operands.iter().enumerate() {
-                    if let InlineAsmOperand::Const { .. } = op {
-                        self.candidates.push(Candidate::InlineAsm { bb: location.block, index })
+
+                if let Some(constant_args) = args_required_const(self.ccx.tcx, def_id) {
+                    for index in constant_args {
+                        self.candidates.push(Candidate::Argument { bb: location.block, index });
                     }
                 }
             }
-            _ => {}
         }
     }
 }
@@ -335,18 +319,6 @@ impl<'tcx> Validator<'_, 'tcx> {
                     _ => bug!(),
                 }
             }
-            Candidate::InlineAsm { bb, index } => {
-                assert!(self.explicit);
-
-                let terminator = self.body[bb].terminator();
-                match &terminator.kind {
-                    TerminatorKind::InlineAsm { operands, .. } => match &operands[index] {
-                        InlineAsmOperand::Const { value } => self.validate_operand(value),
-                        _ => bug!(),
-                    },
-                    _ => bug!(),
-                }
-            }
         }
     }
 
@@ -643,7 +615,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                 self.validate_operand(operand)?;
             }
 
-            Rvalue::BinaryOp(op, lhs, rhs) | Rvalue::CheckedBinaryOp(op, lhs, rhs) => {
+            Rvalue::BinaryOp(op, box (lhs, rhs)) | Rvalue::CheckedBinaryOp(op, box (lhs, rhs)) => {
                 let op = *op;
                 let lhs_ty = lhs.ty(self.body, self.tcx);
 
@@ -818,9 +790,7 @@ pub fn validate_candidates(
             }
 
             match candidate {
-                Candidate::Argument { bb, index } | Candidate::InlineAsm { bb, index }
-                    if !is_promotable =>
-                {
+                Candidate::Argument { bb, index } if !is_promotable => {
                     let span = ccx.body[bb].terminator().source_info.span;
                     let msg = format!("argument {} is required to be a constant", index + 1);
                     ccx.tcx.sess.span_err(span, &msg);
@@ -921,7 +891,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                         let unit = Rvalue::Use(Operand::Constant(box Constant {
                             span: statement.source_info.span,
                             user_ty: None,
-                            literal: ty::Const::zero_sized(self.tcx, self.tcx.types.unit),
+                            literal: ty::Const::zero_sized(self.tcx, self.tcx.types.unit).into(),
                         }));
                         mem::replace(rhs, unit)
                     },
@@ -998,20 +968,22 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                 Operand::Constant(Box::new(Constant {
                     span,
                     user_ty: None,
-                    literal: tcx.mk_const(ty::Const {
-                        ty,
-                        val: ty::ConstKind::Unevaluated(
-                            def,
-                            InternalSubsts::for_item(tcx, def.did, |param, _| {
-                                if let ty::GenericParamDefKind::Lifetime = param.kind {
-                                    tcx.lifetimes.re_erased.into()
-                                } else {
-                                    tcx.mk_param_from_def(param)
-                                }
+                    literal: tcx
+                        .mk_const(ty::Const {
+                            ty,
+                            val: ty::ConstKind::Unevaluated(ty::Unevaluated {
+                                def,
+                                substs: InternalSubsts::for_item(tcx, def.did, |param, _| {
+                                    if let ty::GenericParamDefKind::Lifetime = param.kind {
+                                        tcx.lifetimes.re_erased.into()
+                                    } else {
+                                        tcx.mk_param_from_def(param)
+                                    }
+                                }),
+                                promoted: Some(promoted_id),
                             }),
-                            Some(promoted_id),
-                        ),
-                    }),
+                        })
+                        .into(),
                 }))
             };
             let (blocks, local_decls) = self.source.basic_blocks_and_local_decls_mut();
@@ -1087,24 +1059,6 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                         _ => bug!(),
                     }
                 }
-                Candidate::InlineAsm { bb, index } => {
-                    let terminator = blocks[bb].terminator_mut();
-                    match terminator.kind {
-                        TerminatorKind::InlineAsm { ref mut operands, .. } => {
-                            match &mut operands[index] {
-                                InlineAsmOperand::Const { ref mut value } => {
-                                    let ty = value.ty(local_decls, self.tcx);
-                                    let span = terminator.source_info.span;
-
-                                    Rvalue::Use(mem::replace(value, promoted_operand(ty, span)))
-                                }
-                                _ => bug!(),
-                            }
-                        }
-
-                        _ => bug!(),
-                    }
-                }
             }
         };
 
@@ -1159,7 +1113,7 @@ pub fn promote_candidates<'tcx>(
                     }
                 }
             }
-            Candidate::Argument { .. } | Candidate::InlineAsm { .. } => {}
+            Candidate::Argument { .. } => {}
         }
 
         // Declare return place local so that `mir::Body::new` doesn't complain.
@@ -1250,8 +1204,8 @@ crate fn is_const_fn_in_array_repeat_expression<'tcx>(
         if let Some(Terminator { kind: TerminatorKind::Call { func, destination, .. }, .. }) =
             &block.terminator
         {
-            if let Operand::Constant(box Constant { literal: ty::Const { ty, .. }, .. }) = func {
-                if let ty::FnDef(def_id, _) = *ty.kind() {
+            if let Operand::Constant(box Constant { literal, .. }) = func {
+                if let ty::FnDef(def_id, _) = *literal.ty().kind() {
                     if let Some((destination_place, _)) = destination {
                         if destination_place == place {
                             if is_const_fn(ccx.tcx, def_id) {
diff --git a/compiler/rustc_mir/src/transform/remove_noop_landing_pads.rs b/compiler/rustc_mir/src/transform/remove_noop_landing_pads.rs
index 31e201c3a5b..5347846a4b3 100644
--- a/compiler/rustc_mir/src/transform/remove_noop_landing_pads.rs
+++ b/compiler/rustc_mir/src/transform/remove_noop_landing_pads.rs
@@ -55,6 +55,7 @@ impl RemoveNoopLandingPads {
                 StatementKind::Assign { .. }
                 | StatementKind::SetDiscriminant { .. }
                 | StatementKind::LlvmInlineAsm { .. }
+                | StatementKind::CopyNonOverlapping(..)
                 | StatementKind::Retag { .. } => {
                     return false;
                 }
diff --git a/compiler/rustc_mir/src/transform/remove_zsts.rs b/compiler/rustc_mir/src/transform/remove_zsts.rs
new file mode 100644
index 00000000000..70f7538dd57
--- /dev/null
+++ b/compiler/rustc_mir/src/transform/remove_zsts.rs
@@ -0,0 +1,89 @@
+//! Removes assignments to ZST places.
+
+use crate::transform::MirPass;
+use rustc_middle::mir::tcx::PlaceTy;
+use rustc_middle::mir::{Body, LocalDecls, Place, StatementKind};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+
+pub struct RemoveZsts;
+
+impl<'tcx> MirPass<'tcx> for RemoveZsts {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        if tcx.sess.mir_opt_level() < 3 {
+            return;
+        }
+        let param_env = tcx.param_env(body.source.def_id());
+        let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut();
+        for block in basic_blocks.iter_mut() {
+            for statement in block.statements.iter_mut() {
+                match statement.kind {
+                    StatementKind::Assign(box (place, _)) => {
+                        let place_ty = place.ty(local_decls, tcx).ty;
+                        if !maybe_zst(place_ty) {
+                            continue;
+                        }
+                        let layout = match tcx.layout_of(param_env.and(place_ty)) {
+                            Ok(layout) => layout,
+                            Err(_) => continue,
+                        };
+                        if !layout.is_zst() {
+                            continue;
+                        }
+                        if involves_a_union(place, local_decls, tcx) {
+                            continue;
+                        }
+                        if tcx.consider_optimizing(|| {
+                            format!(
+                                "RemoveZsts - Place: {:?} SourceInfo: {:?}",
+                                place, statement.source_info
+                            )
+                        }) {
+                            statement.make_nop();
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
+    }
+}
+
+/// A cheap, approximate check to avoid unnecessary `layout_of` calls.
+fn maybe_zst(ty: Ty<'_>) -> bool {
+    match ty.kind() {
+        // maybe ZST (could be more precise)
+        ty::Adt(..) | ty::Array(..) | ty::Closure(..) | ty::Tuple(..) | ty::Opaque(..) => true,
+        // definitely ZST
+        ty::FnDef(..) | ty::Never => true,
+        // unreachable or can't be ZST
+        _ => false,
+    }
+}
+
+/// Miri lazily allocates memory for locals on assignment,
+/// so we must preserve writes to unions and union fields,
+/// or it will ICE on reads of those fields.
+fn involves_a_union<'tcx>(
+    place: Place<'tcx>,
+    local_decls: &LocalDecls<'tcx>,
+    tcx: TyCtxt<'tcx>,
+) -> bool {
+    let mut place_ty = PlaceTy::from_ty(local_decls[place.local].ty);
+    if is_union(place_ty.ty) {
+        return true;
+    }
+    for elem in place.projection {
+        place_ty = place_ty.projection_ty(tcx, elem);
+        if is_union(place_ty.ty) {
+            return true;
+        }
+    }
+    return false;
+}
+
+fn is_union(ty: Ty<'_>) -> bool {
+    match ty.kind() {
+        ty::Adt(def, _) if def.is_union() => true,
+        _ => false,
+    }
+}
diff --git a/compiler/rustc_mir/src/transform/required_consts.rs b/compiler/rustc_mir/src/transform/required_consts.rs
index a63ab30a68f..8b64ad65ab3 100644
--- a/compiler/rustc_mir/src/transform/required_consts.rs
+++ b/compiler/rustc_mir/src/transform/required_consts.rs
@@ -14,10 +14,10 @@ impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> {
 
 impl<'a, 'tcx> Visitor<'tcx> for RequiredConstsVisitor<'a, 'tcx> {
     fn visit_constant(&mut self, constant: &Constant<'tcx>, _: Location) {
-        let const_kind = constant.literal.val;
-
-        if let ConstKind::Unevaluated(_, _, _) = const_kind {
-            self.required_consts.push(*constant);
+        if let Some(ct) = constant.literal.const_for_ty() {
+            if let ConstKind::Unevaluated(_) = ct.val {
+                self.required_consts.push(*constant);
+            }
         }
     }
 }
diff --git a/compiler/rustc_mir/src/transform/rustc_peek.rs b/compiler/rustc_mir/src/transform/rustc_peek.rs
index 7598be4e4a1..a6b8f20f6d4 100644
--- a/compiler/rustc_mir/src/transform/rustc_peek.rs
+++ b/compiler/rustc_mir/src/transform/rustc_peek.rs
@@ -205,7 +205,7 @@ impl PeekCall {
         if let mir::TerminatorKind::Call { func: Operand::Constant(func), args, .. } =
             &terminator.kind
         {
-            if let ty::FnDef(def_id, substs) = *func.literal.ty.kind() {
+            if let ty::FnDef(def_id, substs) = *func.literal.ty().kind() {
                 let sig = tcx.fn_sig(def_id);
                 let name = tcx.item_name(def_id);
                 if sig.abi() != Abi::RustIntrinsic || name != sym::rustc_peek {
diff --git a/compiler/rustc_mir/src/transform/simplify.rs b/compiler/rustc_mir/src/transform/simplify.rs
index 85f27428bbb..65e2d096b20 100644
--- a/compiler/rustc_mir/src/transform/simplify.rs
+++ b/compiler/rustc_mir/src/transform/simplify.rs
@@ -414,7 +414,9 @@ impl UsedLocals {
             // A use, not a definition.
             self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location);
         } else {
-            // A definition. Although, it still might use other locals for indexing.
+            // A definition. The base local itself is not visited, so this occurrence is not counted
+            // toward its use count. There might be other locals still, used in an indexing
+            // projection.
             self.super_projection(
                 place.as_ref(),
                 PlaceContext::MutatingUse(MutatingUseContext::Projection),
@@ -428,6 +430,7 @@ impl Visitor<'_> for UsedLocals {
     fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
         match statement.kind {
             StatementKind::LlvmInlineAsm(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::Retag(..)
             | StatementKind::Coverage(..)
             | StatementKind::FakeRead(..)
diff --git a/compiler/rustc_mir/src/transform/simplify_comparison_integral.rs b/compiler/rustc_mir/src/transform/simplify_comparison_integral.rs
index bd76e118fdf..9f473f3bae5 100644
--- a/compiler/rustc_mir/src/transform/simplify_comparison_integral.rs
+++ b/compiler/rustc_mir/src/transform/simplify_comparison_integral.rs
@@ -84,10 +84,10 @@ impl<'tcx> MirPass<'tcx> for SimplifyComparisonIntegral {
 
                 use Operand::*;
                 match rhs {
-                    Rvalue::BinaryOp(_, ref mut left @ Move(_), Constant(_)) => {
+                    Rvalue::BinaryOp(_, box (ref mut left @ Move(_), Constant(_))) => {
                         *left = Copy(opt.to_switch_on);
                     }
-                    Rvalue::BinaryOp(_, Constant(_), ref mut right @ Move(_)) => {
+                    Rvalue::BinaryOp(_, box (Constant(_), ref mut right @ Move(_))) => {
                         *right = Copy(opt.to_switch_on);
                     }
                     _ => (),
@@ -166,7 +166,10 @@ impl<'a, 'tcx> OptimizationFinder<'a, 'tcx> {
                             if *lhs == place_switched_on =>
                         {
                             match rhs {
-                                Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), left, right) => {
+                                Rvalue::BinaryOp(
+                                    op @ (BinOp::Eq | BinOp::Ne),
+                                    box (left, right),
+                                ) => {
                                     let (branch_value_scalar, branch_value_ty, to_switch_on) =
                                         find_branch_value_info(left, right)?;
 
@@ -202,12 +205,12 @@ fn find_branch_value_info<'tcx>(
     match (left, right) {
         (Constant(branch_value), Copy(to_switch_on) | Move(to_switch_on))
         | (Copy(to_switch_on) | Move(to_switch_on), Constant(branch_value)) => {
-            let branch_value_ty = branch_value.literal.ty;
+            let branch_value_ty = branch_value.literal.ty();
             // we only want to apply this optimization if we are matching on integrals (and chars), as it is not possible to switch on floats
             if !branch_value_ty.is_integral() && !branch_value_ty.is_char() {
                 return None;
             };
-            let branch_value_scalar = branch_value.literal.val.try_to_scalar()?;
+            let branch_value_scalar = branch_value.literal.try_to_scalar()?;
             Some((branch_value_scalar, branch_value_ty, *to_switch_on))
         }
         _ => None,
diff --git a/compiler/rustc_mir/src/transform/unreachable_prop.rs b/compiler/rustc_mir/src/transform/unreachable_prop.rs
index e39c8656021..658c6b6e9db 100644
--- a/compiler/rustc_mir/src/transform/unreachable_prop.rs
+++ b/compiler/rustc_mir/src/transform/unreachable_prop.rs
@@ -12,8 +12,8 @@ pub struct UnreachablePropagation;
 
 impl MirPass<'_> for UnreachablePropagation {
     fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        if tcx.sess.opts.debugging_opts.mir_opt_level < 3 {
-            // Enable only under -Zmir-opt-level=3 as in some cases (check the deeply-nested-opt
+        if tcx.sess.mir_opt_level() < 4 {
+            // Enable only under -Zmir-opt-level=4 as in some cases (check the deeply-nested-opt
             // perf benchmark) LLVM may spend quite a lot of time optimizing the generated code.
             return;
         }
diff --git a/compiler/rustc_mir/src/transform/validate.rs b/compiler/rustc_mir/src/transform/validate.rs
index 29b90bff210..d009b0b1b23 100644
--- a/compiler/rustc_mir/src/transform/validate.rs
+++ b/compiler/rustc_mir/src/transform/validate.rs
@@ -294,7 +294,49 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
             }
-            _ => {}
+            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+                ref src,
+                ref dst,
+                ref count,
+            }) => {
+                let src_ty = src.ty(&self.body.local_decls, self.tcx);
+                let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) {
+                    src_deref.ty
+                } else {
+                    self.fail(
+                        location,
+                        format!("Expected src to be ptr in copy_nonoverlapping, got: {}", src_ty),
+                    );
+                    return;
+                };
+                let dst_ty = dst.ty(&self.body.local_decls, self.tcx);
+                let op_dst_ty = if let Some(dst_deref) = dst_ty.builtin_deref(true) {
+                    dst_deref.ty
+                } else {
+                    self.fail(
+                        location,
+                        format!("Expected dst to be ptr in copy_nonoverlapping, got: {}", dst_ty),
+                    );
+                    return;
+                };
+                // since CopyNonOverlapping is parametrized by 1 type,
+                // we only need to check that they are equal and not keep an extra parameter.
+                if op_src_ty != op_dst_ty {
+                    self.fail(location, format!("bad arg ({:?} != {:?})", op_src_ty, op_dst_ty));
+                }
+
+                let op_cnt_ty = count.ty(&self.body.local_decls, self.tcx);
+                if op_cnt_ty != self.tcx.types.usize {
+                    self.fail(location, format!("bad arg ({:?} != usize)", op_cnt_ty))
+                }
+            }
+            StatementKind::SetDiscriminant { .. }
+            | StatementKind::StorageLive(..)
+            | StatementKind::StorageDead(..)
+            | StatementKind::LlvmInlineAsm(..)
+            | StatementKind::Retag(_, _)
+            | StatementKind::Coverage(_)
+            | StatementKind::Nop => {}
         }
 
         self.super_statement(statement, location);
diff --git a/compiler/rustc_mir/src/util/alignment.rs b/compiler/rustc_mir/src/util/alignment.rs
index f567c9cfaab..5d4ca871faa 100644
--- a/compiler/rustc_mir/src/util/alignment.rs
+++ b/compiler/rustc_mir/src/util/alignment.rs
@@ -1,5 +1,6 @@
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
+use rustc_target::abi::Align;
 
 /// Returns `true` if this place is allowed to be less aligned
 /// than its containing struct (because it is within a packed
@@ -14,17 +15,25 @@ where
     L: HasLocalDecls<'tcx>,
 {
     debug!("is_disaligned({:?})", place);
-    if !is_within_packed(tcx, local_decls, place) {
-        debug!("is_disaligned({:?}) - not within packed", place);
-        return false;
-    }
+    let pack = match is_within_packed(tcx, local_decls, place) {
+        None => {
+            debug!("is_disaligned({:?}) - not within packed", place);
+            return false;
+        }
+        Some(pack) => pack,
+    };
 
     let ty = place.ty(local_decls, tcx).ty;
     match tcx.layout_raw(param_env.and(ty)) {
-        Ok(layout) if layout.align.abi.bytes() == 1 => {
-            // if the alignment is 1, the type can't be further
-            // disaligned.
-            debug!("is_disaligned({:?}) - align = 1", place);
+        Ok(layout) if layout.align.abi <= pack => {
+            // If the packed alignment is greater or equal to the field alignment, the type won't be
+            // further disaligned.
+            debug!(
+                "is_disaligned({:?}) - align = {}, packed = {}; not disaligned",
+                place,
+                layout.align.abi.bytes(),
+                pack.bytes()
+            );
             false
         }
         _ => {
@@ -34,7 +43,11 @@ where
     }
 }
 
-fn is_within_packed<'tcx, L>(tcx: TyCtxt<'tcx>, local_decls: &L, place: Place<'tcx>) -> bool
+fn is_within_packed<'tcx, L>(
+    tcx: TyCtxt<'tcx>,
+    local_decls: &L,
+    place: Place<'tcx>,
+) -> Option<Align>
 where
     L: HasLocalDecls<'tcx>,
 {
@@ -45,7 +58,7 @@ where
             ProjectionElem::Field(..) => {
                 let ty = place_base.ty(local_decls, tcx).ty;
                 match ty.kind() {
-                    ty::Adt(def, _) if def.repr.packed() => return true,
+                    ty::Adt(def, _) => return def.repr.pack,
                     _ => {}
                 }
             }
@@ -53,5 +66,5 @@ where
         }
     }
 
-    false
+    None
 }
diff --git a/compiler/rustc_mir/src/util/elaborate_drops.rs b/compiler/rustc_mir/src/util/elaborate_drops.rs
index 0e2d8e5495b..e9190d7ebef 100644
--- a/compiler/rustc_mir/src/util/elaborate_drops.rs
+++ b/compiler/rustc_mir/src/util/elaborate_drops.rs
@@ -678,11 +678,14 @@ where
 
         let one = self.constant_usize(1);
         let (ptr_next, cur_next) = if ptr_based {
-            (Rvalue::Use(copy(cur.into())), Rvalue::BinaryOp(BinOp::Offset, move_(cur.into()), one))
+            (
+                Rvalue::Use(copy(cur.into())),
+                Rvalue::BinaryOp(BinOp::Offset, box (move_(cur.into()), one)),
+            )
         } else {
             (
                 Rvalue::AddressOf(Mutability::Mut, tcx.mk_place_index(self.place, cur)),
-                Rvalue::BinaryOp(BinOp::Add, move_(cur.into()), one),
+                Rvalue::BinaryOp(BinOp::Add, box (move_(cur.into()), one)),
             )
         };
 
@@ -700,7 +703,7 @@ where
         let loop_block = BasicBlockData {
             statements: vec![self.assign(
                 can_go,
-                Rvalue::BinaryOp(BinOp::Eq, copy(Place::from(cur)), copy(length_or_end)),
+                Rvalue::BinaryOp(BinOp::Eq, box (copy(Place::from(cur)), copy(length_or_end))),
             )],
             is_cleanup: unwind.is_cleanup(),
             terminator: Some(Terminator {
@@ -816,7 +819,10 @@ where
                 self.assign(cur, Rvalue::Cast(CastKind::Misc, Operand::Move(tmp), iter_ty)),
                 self.assign(
                     length_or_end,
-                    Rvalue::BinaryOp(BinOp::Offset, Operand::Copy(cur), Operand::Move(length)),
+                    Rvalue::BinaryOp(
+                        BinOp::Offset,
+                        box (Operand::Copy(cur), Operand::Move(length)),
+                    ),
                 ),
             ]
         } else {
@@ -1029,7 +1035,7 @@ where
         Operand::Constant(box Constant {
             span: self.source_info.span,
             user_ty: None,
-            literal: ty::Const::from_usize(self.tcx(), val.into()),
+            literal: ty::Const::from_usize(self.tcx(), val.into()).into(),
         })
     }
 
diff --git a/compiler/rustc_mir/src/util/find_self_call.rs b/compiler/rustc_mir/src/util/find_self_call.rs
index 5b146eeb87c..33ad128eeeb 100644
--- a/compiler/rustc_mir/src/util/find_self_call.rs
+++ b/compiler/rustc_mir/src/util/find_self_call.rs
@@ -3,7 +3,7 @@ use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::def_id::DefId;
 
-/// Checks if the specified `local` is used as the `self` prameter of a method call
+/// Checks if the specified `local` is used as the `self` parameter of a method call
 /// in the provided `BasicBlock`. If it is, then the `DefId` of the called method is
 /// returned.
 pub fn find_self_call<'tcx>(
@@ -17,8 +17,8 @@ pub fn find_self_call<'tcx>(
         &body[block].terminator
     {
         debug!("find_self_call: func={:?}", func);
-        if let Operand::Constant(box Constant { literal: ty::Const { ty, .. }, .. }) = func {
-            if let ty::FnDef(def_id, substs) = *ty.kind() {
+        if let Operand::Constant(box Constant { literal, .. }) = func {
+            if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
                 if let Some(ty::AssocItem { fn_has_self_parameter: true, .. }) =
                     tcx.opt_associated_item(def_id)
                 {
diff --git a/compiler/rustc_mir/src/util/generic_graphviz.rs b/compiler/rustc_mir/src/util/generic_graphviz.rs
index fd41e282266..21c18b28e25 100644
--- a/compiler/rustc_mir/src/util/generic_graphviz.rs
+++ b/compiler/rustc_mir/src/util/generic_graphviz.rs
@@ -40,22 +40,6 @@ impl<
         }
     }
 
-    pub fn new_subgraph(
-        graph: &'a G,
-        graphviz_name: &str,
-        node_content_fn: NodeContentFn,
-        edge_labels_fn: EdgeLabelsFn,
-    ) -> Self {
-        Self {
-            graph,
-            is_subgraph: true,
-            graphviz_name: graphviz_name.to_owned(),
-            graph_label: None,
-            node_content_fn,
-            edge_labels_fn,
-        }
-    }
-
     pub fn set_graph_label(&mut self, graph_label: &str) {
         self.graph_label = Some(graph_label.to_owned());
     }
diff --git a/compiler/rustc_mir/src/util/patch.rs b/compiler/rustc_mir/src/util/patch.rs
index 6566a996fe4..d09195f53ae 100644
--- a/compiler/rustc_mir/src/util/patch.rs
+++ b/compiler/rustc_mir/src/util/patch.rs
@@ -13,7 +13,6 @@ pub struct MirPatch<'tcx> {
     new_locals: Vec<LocalDecl<'tcx>>,
     resume_block: BasicBlock,
     next_local: usize,
-    make_nop: Vec<Location>,
 }
 
 impl<'tcx> MirPatch<'tcx> {
@@ -25,7 +24,6 @@ impl<'tcx> MirPatch<'tcx> {
             new_locals: vec![],
             next_local: body.local_decls.len(),
             resume_block: START_BLOCK,
-            make_nop: vec![],
         };
 
         // make sure the MIR we create has a resume block. It is
@@ -117,15 +115,7 @@ impl<'tcx> MirPatch<'tcx> {
         self.add_statement(loc, StatementKind::Assign(box (place, rv)));
     }
 
-    pub fn make_nop(&mut self, loc: Location) {
-        self.make_nop.push(loc);
-    }
-
     pub fn apply(self, body: &mut Body<'tcx>) {
-        debug!("MirPatch: make nops at: {:?}", self.make_nop);
-        for loc in self.make_nop {
-            body.make_statement_nop(loc);
-        }
         debug!(
             "MirPatch: {:?} new temps, starting from index {}: {:?}",
             self.new_locals.len(),
diff --git a/compiler/rustc_mir/src/util/pretty.rs b/compiler/rustc_mir/src/util/pretty.rs
index 247a0beccaf..3b88aec16b2 100644
--- a/compiler/rustc_mir/src/util/pretty.rs
+++ b/compiler/rustc_mir/src/util/pretty.rs
@@ -439,7 +439,7 @@ impl Visitor<'tcx> for ExtraComments<'tcx> {
     fn visit_constant(&mut self, constant: &Constant<'tcx>, location: Location) {
         self.super_constant(constant, location);
         let Constant { span, user_ty, literal } = constant;
-        match literal.ty.kind() {
+        match literal.ty().kind() {
             ty::Int(_) | ty::Uint(_) | ty::Bool | ty::Char => {}
             // Unit type
             ty::Tuple(tys) if tys.is_empty() => {}
@@ -449,7 +449,16 @@ impl Visitor<'tcx> for ExtraComments<'tcx> {
                 if let Some(user_ty) = user_ty {
                     self.push(&format!("+ user_ty: {:?}", user_ty));
                 }
-                self.push(&format!("+ literal: {:?}", literal));
+                match literal {
+                    ConstantKind::Ty(literal) => self.push(&format!("+ literal: {:?}", literal)),
+                    ConstantKind::Val(val, ty) => {
+                        // To keep the diffs small, we render this almost like we render ty::Const
+                        self.push(&format!(
+                            "+ literal: Const {{ ty: {}, val: Value({:?}) }}",
+                            ty, val
+                        ))
+                    }
+                }
             }
         }
     }
@@ -460,7 +469,21 @@ impl Visitor<'tcx> for ExtraComments<'tcx> {
         if use_verbose(ty) {
             self.push("ty::Const");
             self.push(&format!("+ ty: {:?}", ty));
-            self.push(&format!("+ val: {:?}", val));
+            let val = match val {
+                ty::ConstKind::Param(p) => format!("Param({})", p),
+                ty::ConstKind::Infer(infer) => format!("Infer({:?})", infer),
+                ty::ConstKind::Bound(idx, var) => format!("Bound({:?}, {:?})", idx, var),
+                ty::ConstKind::Placeholder(ph) => format!("PlaceHolder({:?})", ph),
+                ty::ConstKind::Unevaluated(uv) => format!(
+                    "Unevaluated({}, {:?}, {:?})",
+                    self.tcx.def_path_str(uv.def.did),
+                    uv.substs,
+                    uv.promoted
+                ),
+                ty::ConstKind::Value(val) => format!("Value({:?})", val),
+                ty::ConstKind::Error(_) => format!("Error"),
+            };
+            self.push(&format!("+ val: {}", val));
         }
     }
 
diff --git a/compiler/rustc_mir/src/util/spanview.rs b/compiler/rustc_mir/src/util/spanview.rs
index d3ef8c64565..a9a30e407b4 100644
--- a/compiler/rustc_mir/src/util/spanview.rs
+++ b/compiler/rustc_mir/src/util/spanview.rs
@@ -245,6 +245,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str {
         Retag(..) => "Retag",
         AscribeUserType(..) => "AscribeUserType",
         Coverage(..) => "Coverage",
+        CopyNonOverlapping(..) => "CopyNonOverlapping",
         Nop => "Nop",
     }
 }
diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
index d5f72e6f22d..808c6e3ff64 100644
--- a/compiler/rustc_mir_build/src/build/block.rs
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -2,7 +2,6 @@ use crate::build::matches::ArmHasGuard;
 use crate::build::ForGuard::OutsideGuard;
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
 use crate::thir::*;
-use rustc_hir as hir;
 use rustc_middle::mir::*;
 use rustc_session::lint::builtin::UNSAFE_OP_IN_UNSAFE_FN;
 use rustc_session::lint::Level;
@@ -13,7 +12,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         destination: Place<'tcx>,
         block: BasicBlock,
-        ast_block: &'tcx hir::Block<'tcx>,
+        ast_block: &Block<'_, 'tcx>,
         source_info: SourceInfo,
     ) -> BlockAnd<()> {
         let Block {
@@ -24,7 +23,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             expr,
             targeted_by_break,
             safety_mode,
-        } = self.hir.mirror(ast_block);
+        } = *ast_block;
         self.in_opt_scope(opt_destruction_scope.map(|de| (de, source_info)), move |this| {
             this.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
                 if targeted_by_break {
@@ -50,8 +49,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         destination: Place<'tcx>,
         mut block: BasicBlock,
         span: Span,
-        stmts: Vec<StmtRef<'tcx>>,
-        expr: Option<ExprRef<'tcx>>,
+        stmts: &[Stmt<'_, 'tcx>],
+        expr: Option<&Expr<'_, 'tcx>>,
         safety_mode: BlockSafety,
     ) -> BlockAnd<()> {
         let this = self;
@@ -79,10 +78,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         this.update_source_scope_for_safety_mode(span, safety_mode);
 
         let source_info = this.source_info(span);
-        for stmt in stmts {
-            let Stmt { kind, opt_destruction_scope } = this.hir.mirror(stmt);
+        for Stmt { kind, opt_destruction_scope } in stmts {
             match kind {
-                StmtKind::Expr { scope, expr } => {
+                &StmtKind::Expr { scope, expr } => {
                     this.block_context.push(BlockFrame::Statement { ignores_expr_result: true });
                     unpack!(
                         block = this.in_opt_scope(
@@ -90,7 +88,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             |this| {
                                 let si = (scope, source_info);
                                 this.in_scope(si, LintLevel::Inherited, |this| {
-                                    let expr = this.hir.mirror(expr);
                                     this.stmt_expr(block, expr, Some(scope))
                                 })
                             }
@@ -102,45 +99,44 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     this.block_context.push(BlockFrame::Statement { ignores_expr_result });
 
                     // Enter the remainder scope, i.e., the bindings' destruction scope.
-                    this.push_scope((remainder_scope, source_info));
+                    this.push_scope((*remainder_scope, source_info));
                     let_scope_stack.push(remainder_scope);
 
                     // Declare the bindings, which may create a source scope.
-                    let remainder_span =
-                        remainder_scope.span(this.hir.tcx(), &this.hir.region_scope_tree);
+                    let remainder_span = remainder_scope.span(this.tcx, this.region_scope_tree);
 
                     let visibility_scope =
                         Some(this.new_source_scope(remainder_span, LintLevel::Inherited, None));
 
                     // Evaluate the initializer, if present.
                     if let Some(init) = initializer {
-                        let initializer_span = init.span();
+                        let initializer_span = init.span;
 
                         unpack!(
                             block = this.in_opt_scope(
                                 opt_destruction_scope.map(|de| (de, source_info)),
                                 |this| {
-                                    let scope = (init_scope, source_info);
-                                    this.in_scope(scope, lint_level, |this| {
+                                    let scope = (*init_scope, source_info);
+                                    this.in_scope(scope, *lint_level, |this| {
                                         this.declare_bindings(
                                             visibility_scope,
                                             remainder_span,
-                                            &pattern,
+                                            pattern,
                                             ArmHasGuard(false),
                                             Some((None, initializer_span)),
                                         );
-                                        this.expr_into_pattern(block, pattern, init)
+                                        this.expr_into_pattern(block, pattern.clone(), init)
                                     })
                                 }
                             )
                         );
                     } else {
-                        let scope = (init_scope, source_info);
-                        unpack!(this.in_scope(scope, lint_level, |this| {
+                        let scope = (*init_scope, source_info);
+                        unpack!(this.in_scope(scope, *lint_level, |this| {
                             this.declare_bindings(
                                 visibility_scope,
                                 remainder_span,
-                                &pattern,
+                                pattern,
                                 ArmHasGuard(false),
                                 None,
                             );
@@ -171,18 +167,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         // Then, the block may have an optional trailing expression which is a “return” value
         // of the block, which is stored into `destination`.
-        let tcx = this.hir.tcx();
+        let tcx = this.tcx;
         let destination_ty = destination.ty(&this.local_decls, tcx).ty;
         if let Some(expr) = expr {
             let tail_result_is_ignored =
                 destination_ty.is_unit() || this.block_context.currently_ignores_tail_results();
-            let span = match expr {
-                ExprRef::Thir(expr) => expr.span,
-                ExprRef::Mirror(ref expr) => expr.span,
-            };
-            this.block_context.push(BlockFrame::TailExpr { tail_result_is_ignored, span });
+            this.block_context
+                .push(BlockFrame::TailExpr { tail_result_is_ignored, span: expr.span });
 
-            unpack!(block = this.into(destination, block, expr));
+            unpack!(block = this.expr_into_dest(destination, block, expr));
             let popped = this.block_context.pop();
 
             assert!(popped.map_or(false, |bf| bf.is_tail_expr()));
@@ -194,13 +187,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             if destination_ty.is_unit() {
                 // We only want to assign an implicit `()` as the return value of the block if the
                 // block does not diverge. (Otherwise, we may try to assign a unit to a `!`-type.)
-                this.cfg.push_assign_unit(block, source_info, destination, this.hir.tcx());
+                this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
             }
         }
         // Finally, we pop all the let scopes before exiting out from the scope of block
         // itself.
         for scope in let_scope_stack.into_iter().rev() {
-            unpack!(block = this.pop_scope((scope, source_info), block));
+            unpack!(block = this.pop_scope((*scope, source_info), block));
         }
         // Restore the original source scope.
         this.source_scope = outer_source_scope;
@@ -220,7 +213,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     Safety::Safe => {}
                     // no longer treat `unsafe fn`s as `unsafe` contexts (see RFC #2585)
                     Safety::FnUnsafe
-                        if self.hir.tcx().lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, hir_id).0
+                        if self.tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, hir_id).0
                             != Level::Allow => {}
                     _ => return,
                 }
diff --git a/compiler/rustc_mir_build/src/build/cfg.rs b/compiler/rustc_mir_build/src/build/cfg.rs
index 42e2b242d77..fd4a783d12a 100644
--- a/compiler/rustc_mir_build/src/build/cfg.rs
+++ b/compiler/rustc_mir_build/src/build/cfg.rs
@@ -68,7 +68,7 @@ impl<'tcx> CFG<'tcx> {
             Rvalue::Use(Operand::Constant(box Constant {
                 span: source_info.span,
                 user_ty: None,
-                literal: ty::Const::zero_sized(tcx, tcx.types.unit),
+                literal: ty::Const::zero_sized(tcx, tcx.types.unit).into(),
             })),
         );
     }
@@ -80,7 +80,7 @@ impl<'tcx> CFG<'tcx> {
         cause: FakeReadCause,
         place: Place<'tcx>,
     ) {
-        let kind = StatementKind::FakeRead(cause, box place);
+        let kind = StatementKind::FakeRead(box (cause, place));
         let stmt = Statement { source_info, kind };
         self.push(block, stmt);
     }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index 3a36ad590c5..57f56e2d092 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -8,18 +8,10 @@ use rustc_middle::ty::CanonicalUserTypeAnnotation;
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, yielding a compile-time constant. Assumes that
     /// `expr` is a valid compile-time constant!
-    crate fn as_constant<M>(&mut self, expr: M) -> Constant<'tcx>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
-        let expr = self.hir.mirror(expr);
-        self.expr_as_constant(expr)
-    }
-
-    fn expr_as_constant(&mut self, expr: Expr<'tcx>) -> Constant<'tcx> {
+    crate fn as_constant(&mut self, expr: &Expr<'_, 'tcx>) -> Constant<'tcx> {
         let this = self;
-        let Expr { ty, temp_lifetime: _, span, kind } = expr;
-        match kind {
+        let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
+        match *kind {
             ExprKind::Scope { region_scope: _, lint_level: _, value } => this.as_constant(value),
             ExprKind::Literal { literal, user_ty, const_id: _ } => {
                 let user_ty = user_ty.map(|user_ty| {
@@ -30,10 +22,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     })
                 });
                 assert_eq!(literal.ty, ty);
-                Constant { span, user_ty, literal }
+                Constant { span, user_ty, literal: literal.into() }
+            }
+            ExprKind::StaticRef { literal, .. } => {
+                Constant { span, user_ty: None, literal: literal.into() }
+            }
+            ExprKind::ConstBlock { value } => {
+                Constant { span: span, user_ty: None, literal: value.into() }
             }
-            ExprKind::StaticRef { literal, .. } => Constant { span, user_ty: None, literal },
-            ExprKind::ConstBlock { value } => Constant { span, user_ty: None, literal: value },
             _ => span_bug!(span, "expression is not a valid constant {:?}", kind),
         }
     }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
index 60f8d8c8a9f..c393878e0b9 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_operand.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
@@ -14,10 +14,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// after the current enclosing `ExprKind::Scope` has ended, so
     /// please do *not* return it from functions to avoid bad
     /// miscompiles.
-    crate fn as_local_operand<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Operand<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
+    crate fn as_local_operand(
+        &mut self,
+        block: BasicBlock,
+        expr: &Expr<'_, 'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
         let local_scope = self.local_scope();
         self.as_operand(block, Some(local_scope), expr)
     }
@@ -70,14 +71,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// value to the stack.
     ///
     /// See #68034 for more details.
-    crate fn as_local_call_operand<M>(
+    crate fn as_local_call_operand(
         &mut self,
         block: BasicBlock,
-        expr: M,
-    ) -> BlockAnd<Operand<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
+        expr: &Expr<'_, 'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
         let local_scope = self.local_scope();
         self.as_call_operand(block, Some(local_scope), expr)
     }
@@ -88,41 +86,16 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// this time.
     ///
     /// The operand is known to be live until the end of `scope`.
-    crate fn as_operand<M>(
-        &mut self,
-        block: BasicBlock,
-        scope: Option<region::Scope>,
-        expr: M,
-    ) -> BlockAnd<Operand<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
-        let expr = self.hir.mirror(expr);
-        self.expr_as_operand(block, scope, expr)
-    }
-
+    ///
     /// Like `as_local_call_operand`, except that the argument will
     /// not be valid once `scope` ends.
-    fn as_call_operand<M>(
-        &mut self,
-        block: BasicBlock,
-        scope: Option<region::Scope>,
-        expr: M,
-    ) -> BlockAnd<Operand<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
-        let expr = self.hir.mirror(expr);
-        self.expr_as_call_operand(block, scope, expr)
-    }
-
-    fn expr_as_operand(
+    crate fn as_operand(
         &mut self,
         mut block: BasicBlock,
         scope: Option<region::Scope>,
-        expr: Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
     ) -> BlockAnd<Operand<'tcx>> {
-        debug!("expr_as_operand(block={:?}, expr={:?})", block, expr);
+        debug!("as_operand(block={:?}, expr={:?})", block, expr);
         let this = self;
 
         if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
@@ -133,7 +106,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
 
         let category = Category::of(&expr.kind).unwrap();
-        debug!("expr_as_operand: category={:?} for={:?}", category, expr.kind);
+        debug!("as_operand: category={:?} for={:?}", category, expr.kind);
         match category {
             Category::Constant => {
                 let constant = this.as_constant(expr);
@@ -146,13 +119,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
-    fn expr_as_call_operand(
+    crate fn as_call_operand(
         &mut self,
         mut block: BasicBlock,
         scope: Option<region::Scope>,
-        expr: Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
     ) -> BlockAnd<Operand<'tcx>> {
-        debug!("expr_as_call_operand(block={:?}, expr={:?})", block, expr);
+        debug!("as_call_operand(block={:?}, expr={:?})", block, expr);
         let this = self;
 
         if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
@@ -163,12 +136,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             });
         }
 
-        let tcx = this.hir.tcx();
+        let tcx = this.tcx;
 
         if tcx.features().unsized_fn_params {
             let ty = expr.ty;
             let span = expr.span;
-            let param_env = this.hir.param_env;
+            let param_env = this.param_env;
 
             if !ty.is_sized(tcx.at(span), param_env) {
                 // !sized means !copy, so this is an unsized move
@@ -176,9 +149,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 // As described above, detect the case where we are passing a value of unsized
                 // type, and that value is coming from the deref of a box.
-                if let ExprKind::Deref { ref arg } = expr.kind {
-                    let arg = this.hir.mirror(arg.clone());
-
+                if let ExprKind::Deref { arg } = expr.kind {
                     // Generate let tmp0 = arg0
                     let operand = unpack!(block = this.as_temp(block, scope, arg, Mutability::Mut));
 
@@ -193,6 +164,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
         }
 
-        this.expr_as_operand(block, scope, expr)
+        this.as_operand(block, scope, expr)
     }
 }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index 3308a243a3a..1053890e618 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -10,14 +10,17 @@ use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
 use rustc_middle::middle::region;
 use rustc_middle::mir::AssertKind::BoundsCheck;
 use rustc_middle::mir::*;
+use rustc_middle::ty::AdtDef;
 use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, Variance};
 use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
 
 use rustc_index::vec::Idx;
 
+use std::iter;
+
 /// The "outermost" place that holds this value.
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug, PartialEq)]
 crate enum PlaceBase {
     /// Denotes the start of a `Place`.
     Local(Local),
@@ -67,7 +70,7 @@ crate enum PlaceBase {
 ///
 /// This is used internally when building a place for an expression like `a.b.c`. The fields `b`
 /// and `c` can be progressively pushed onto the place builder that is created when converting `a`.
-#[derive(Clone)]
+#[derive(Clone, Debug, PartialEq)]
 crate struct PlaceBuilder<'tcx> {
     base: PlaceBase,
     projection: Vec<PlaceElem<'tcx>>,
@@ -77,26 +80,29 @@ crate struct PlaceBuilder<'tcx> {
 /// The projections are truncated to represent a path that might be captured by a
 /// closure/generator. This implies the vector returned from this function doesn't contain
 /// ProjectionElems `Downcast`, `ConstantIndex`, `Index`, or `Subslice` because those will never be
-/// part of a path that is captued by a closure. We stop applying projections once we see the first
+/// part of a path that is captured by a closure. We stop applying projections once we see the first
 /// projection that isn't captured by a closure.
 fn convert_to_hir_projections_and_truncate_for_capture<'tcx>(
     mir_projections: &[PlaceElem<'tcx>],
 ) -> Vec<HirProjectionKind> {
     let mut hir_projections = Vec::new();
+    let mut variant = None;
 
     for mir_projection in mir_projections {
         let hir_projection = match mir_projection {
             ProjectionElem::Deref => HirProjectionKind::Deref,
             ProjectionElem::Field(field, _) => {
-                // We will never encouter this for multivariant enums,
-                // read the comment for `Downcast`.
-                HirProjectionKind::Field(field.index() as u32, VariantIdx::new(0))
+                let variant = variant.unwrap_or(VariantIdx::new(0));
+                HirProjectionKind::Field(field.index() as u32, variant)
             }
-            ProjectionElem::Downcast(..) => {
-                // This projections exist only for enums that have
-                // multiple variants. Since such enums that are captured
-                // completely, we can stop here.
-                break;
+            ProjectionElem::Downcast(.., idx) => {
+                // We don't expect to see multi-variant enums here, as earlier
+                // phases will have truncated them already. However, there can
+                // still be downcasts, thanks to single-variant enums.
+                // We keep track of VariantIdx so we can use this information
+                // if the next ProjectionElem is a Field.
+                variant = Some(*idx);
+                continue;
             }
             ProjectionElem::Index(..)
             | ProjectionElem::ConstantIndex { .. }
@@ -106,7 +112,7 @@ fn convert_to_hir_projections_and_truncate_for_capture<'tcx>(
                 break;
             }
         };
-
+        variant = None;
         hir_projections.push(hir_projection);
     }
 
@@ -136,7 +142,7 @@ fn is_ancestor_or_same_capture(
         return false;
     }
 
-    proj_possible_ancestor.iter().zip(proj_capture).all(|(a, b)| a == b)
+    iter::zip(proj_possible_ancestor, proj_capture).all(|(a, b)| a == b)
 }
 
 /// Computes the index of a capture within the desugared closure provided the closure's
@@ -194,12 +200,12 @@ fn find_capture_matching_projections<'a, 'tcx>(
 /// Takes a PlaceBuilder and resolves the upvar (if any) within it, so that the
 /// `PlaceBuilder` now starts from `PlaceBase::Local`.
 ///
-/// Returns a Result with the error being the HirId of the Upvar that was not found.
+/// Returns a Result with the error being the PlaceBuilder (`from_builder`) that was not found.
 fn to_upvars_resolved_place_builder<'a, 'tcx>(
     from_builder: PlaceBuilder<'tcx>,
     tcx: TyCtxt<'tcx>,
     typeck_results: &'a ty::TypeckResults<'tcx>,
-) -> Result<PlaceBuilder<'tcx>, HirId> {
+) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
     match from_builder.base {
         PlaceBase::Local(_) => Ok(from_builder),
         PlaceBase::Upvar { var_hir_id, closure_def_id, closure_kind } => {
@@ -230,13 +236,12 @@ fn to_upvars_resolved_place_builder<'a, 'tcx>(
                         from_builder.projection
                     )
                 } else {
-                    // FIXME(project-rfc-2229#24): Handle this case properly
                     debug!(
                         "No associated capture found for {:?}[{:#?}]",
                         var_hir_id, from_builder.projection,
                     );
                 }
-                return Err(var_hir_id);
+                return Err(from_builder);
             };
 
             let closure_ty = typeck_results
@@ -300,6 +305,25 @@ impl<'tcx> PlaceBuilder<'tcx> {
         to_upvars_resolved_place_builder(self, tcx, typeck_results).unwrap()
     }
 
+    /// Attempts to resolve the `PlaceBuilder`.
+    /// On success, it will return the resolved `PlaceBuilder`.
+    /// On failure, it will return itself.
+    ///
+    /// Upvars resolve may fail for a `PlaceBuilder` when attempting to
+    /// resolve a disjoint field whose root variable is not captured
+    /// (destructured assignments) or when attempting to resolve a root
+    /// variable (discriminant matching with only wildcard arm) that is
+    /// not captured. This can happen because the final mir that will be
+    /// generated doesn't require a read for this place. Failures will only
+    /// happen inside closures.
+    crate fn try_upvars_resolved<'a>(
+        self,
+        tcx: TyCtxt<'tcx>,
+        typeck_results: &'a ty::TypeckResults<'tcx>,
+    ) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
+        to_upvars_resolved_place_builder(self, tcx, typeck_results)
+    }
+
     crate fn base(&self) -> PlaceBase {
         self.base
     }
@@ -308,15 +332,22 @@ impl<'tcx> PlaceBuilder<'tcx> {
         self.project(PlaceElem::Field(f, ty))
     }
 
-    fn deref(self) -> Self {
+    crate fn deref(self) -> Self {
         self.project(PlaceElem::Deref)
     }
 
+    crate fn downcast(self, adt_def: &'tcx AdtDef, variant_index: VariantIdx) -> Self {
+        self.project(PlaceElem::Downcast(
+            Some(adt_def.variants[variant_index].ident.name),
+            variant_index,
+        ))
+    }
+
     fn index(self, index: Local) -> Self {
         self.project(PlaceElem::Index(index))
     }
 
-    fn project(mut self, elem: PlaceElem<'tcx>) -> Self {
+    crate fn project(mut self, elem: PlaceElem<'tcx>) -> Self {
         self.projection.push(elem);
         self
     }
@@ -347,25 +378,22 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Extra care is needed if any user code is allowed to run between calling
     /// this method and using it, as is the case for `match` and index
     /// expressions.
-    crate fn as_place<M>(&mut self, mut block: BasicBlock, expr: M) -> BlockAnd<Place<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
+    crate fn as_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr: &Expr<'_, 'tcx>,
+    ) -> BlockAnd<Place<'tcx>> {
         let place_builder = unpack!(block = self.as_place_builder(block, expr));
-        block.and(place_builder.into_place(self.hir.tcx(), self.hir.typeck_results()))
+        block.and(place_builder.into_place(self.tcx, self.typeck_results))
     }
 
     /// This is used when constructing a compound `Place`, so that we can avoid creating
     /// intermediate `Place` values until we know the full set of projections.
-    crate fn as_place_builder<M>(
+    crate fn as_place_builder(
         &mut self,
         block: BasicBlock,
-        expr: M,
-    ) -> BlockAnd<PlaceBuilder<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
-        let expr = self.hir.mirror(expr);
+        expr: &Expr<'_, 'tcx>,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
         self.expr_as_place(block, expr, Mutability::Mut, None)
     }
 
@@ -374,16 +402,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// place. The place itself may or may not be mutable:
     /// * If this expr is a place expr like a.b, then we will return that place.
     /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
-    crate fn as_read_only_place<M>(
+    crate fn as_read_only_place(
         &mut self,
         mut block: BasicBlock,
-        expr: M,
-    ) -> BlockAnd<Place<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
+        expr: &Expr<'_, 'tcx>,
+    ) -> BlockAnd<Place<'tcx>> {
         let place_builder = unpack!(block = self.as_read_only_place_builder(block, expr));
-        block.and(place_builder.into_place(self.hir.tcx(), self.hir.typeck_results()))
+        block.and(place_builder.into_place(self.tcx, self.typeck_results))
     }
 
     /// This is used when constructing a compound `Place`, so that we can avoid creating
@@ -392,22 +417,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// place. The place itself may or may not be mutable:
     /// * If this expr is a place expr like a.b, then we will return that place.
     /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
-    fn as_read_only_place_builder<M>(
+    fn as_read_only_place_builder(
         &mut self,
         block: BasicBlock,
-        expr: M,
-    ) -> BlockAnd<PlaceBuilder<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
-        let expr = self.hir.mirror(expr);
+        expr: &Expr<'_, 'tcx>,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
         self.expr_as_place(block, expr, Mutability::Not, None)
     }
 
     fn expr_as_place(
         &mut self,
         mut block: BasicBlock,
-        expr: Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
         mutability: Mutability,
         fake_borrow_temps: Option<&mut Vec<Local>>,
     ) -> BlockAnd<PlaceBuilder<'tcx>> {
@@ -419,18 +440,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match expr.kind {
             ExprKind::Scope { region_scope, lint_level, value } => {
                 this.in_scope((region_scope, source_info), lint_level, |this| {
-                    let value = this.hir.mirror(value);
                     this.expr_as_place(block, value, mutability, fake_borrow_temps)
                 })
             }
             ExprKind::Field { lhs, name } => {
-                let lhs = this.hir.mirror(lhs);
                 let place_builder =
                     unpack!(block = this.expr_as_place(block, lhs, mutability, fake_borrow_temps,));
                 block.and(place_builder.field(name, expr.ty))
             }
             ExprKind::Deref { arg } => {
-                let arg = this.hir.mirror(arg);
                 let place_builder =
                     unpack!(block = this.expr_as_place(block, arg, mutability, fake_borrow_temps,));
                 block.and(place_builder.deref())
@@ -462,7 +480,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             ExprKind::PlaceTypeAscription { source, user_ty } => {
-                let source = this.hir.mirror(source);
                 let place_builder = unpack!(
                     block = this.expr_as_place(block, source, mutability, fake_borrow_temps,)
                 );
@@ -474,8 +491,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             inferred_ty: expr.ty,
                         });
 
-                    let place =
-                        place_builder.clone().into_place(this.hir.tcx(), this.hir.typeck_results());
+                    let place = place_builder.clone().into_place(this.tcx, this.typeck_results);
                     this.cfg.push(
                         block,
                         Statement {
@@ -493,7 +509,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 block.and(place_builder)
             }
             ExprKind::ValueTypeAscription { source, user_ty } => {
-                let source = this.hir.mirror(source);
                 let temp =
                     unpack!(block = this.as_temp(block, source.temp_lifetime, source, mutability));
                 if let Some(user_ty) = user_ty {
@@ -563,19 +578,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     /// Lower a captured upvar. Note we might not know the actual capture index,
     /// so we create a place starting from `PlaceBase::Upvar`, which will be resolved
-    /// once all projections that allow us to indentify a capture have been applied.
+    /// once all projections that allow us to identify a capture have been applied.
     fn lower_captured_upvar(
         &mut self,
         block: BasicBlock,
         upvar_id: ty::UpvarId,
     ) -> BlockAnd<PlaceBuilder<'tcx>> {
         let closure_ty = self
-            .hir
-            .typeck_results()
-            .node_type(self.hir.tcx().hir().local_def_id_to_hir_id(upvar_id.closure_expr_id));
+            .typeck_results
+            .node_type(self.tcx.hir().local_def_id_to_hir_id(upvar_id.closure_expr_id));
 
         let closure_kind = if let ty::Closure(_, closure_substs) = closure_ty.kind() {
-            self.hir.infcx().closure_kind(closure_substs).unwrap()
+            self.infcx.closure_kind(closure_substs).unwrap()
         } else {
             // Generators are considered FnOnce.
             ty::ClosureKind::FnOnce
@@ -599,41 +613,32 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn lower_index_expression(
         &mut self,
         mut block: BasicBlock,
-        base: ExprRef<'tcx>,
-        index: ExprRef<'tcx>,
+        base: &Expr<'_, 'tcx>,
+        index: &Expr<'_, 'tcx>,
         mutability: Mutability,
         fake_borrow_temps: Option<&mut Vec<Local>>,
         temp_lifetime: Option<region::Scope>,
         expr_span: Span,
         source_info: SourceInfo,
     ) -> BlockAnd<PlaceBuilder<'tcx>> {
-        let lhs = self.hir.mirror(base);
-
         let base_fake_borrow_temps = &mut Vec::new();
         let is_outermost_index = fake_borrow_temps.is_none();
         let fake_borrow_temps = fake_borrow_temps.unwrap_or(base_fake_borrow_temps);
 
         let mut base_place =
-            unpack!(block = self.expr_as_place(block, lhs, mutability, Some(fake_borrow_temps),));
+            unpack!(block = self.expr_as_place(block, base, mutability, Some(fake_borrow_temps),));
 
         // Making this a *fresh* temporary means we do not have to worry about
         // the index changing later: Nothing will ever change this temporary.
         // The "retagging" transformation (for Stacked Borrows) relies on this.
         let idx = unpack!(block = self.as_temp(block, temp_lifetime, index, Mutability::Not,));
 
-        block = self.bounds_check(
-            block,
-            base_place.clone().into_place(self.hir.tcx(), self.hir.typeck_results()),
-            idx,
-            expr_span,
-            source_info,
-        );
+        block = self.bounds_check(block, base_place.clone(), idx, expr_span, source_info);
 
         if is_outermost_index {
             self.read_fake_borrows(block, fake_borrow_temps, source_info)
         } else {
-            base_place =
-                base_place.expect_upvars_resolved(self.hir.tcx(), self.hir.typeck_results());
+            base_place = base_place.expect_upvars_resolved(self.tcx, self.typeck_results);
             self.add_fake_borrows_of_base(
                 &base_place,
                 block,
@@ -649,25 +654,33 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn bounds_check(
         &mut self,
         block: BasicBlock,
-        slice: Place<'tcx>,
+        slice: PlaceBuilder<'tcx>,
         index: Local,
         expr_span: Span,
         source_info: SourceInfo,
     ) -> BasicBlock {
-        let usize_ty = self.hir.usize_ty();
-        let bool_ty = self.hir.bool_ty();
+        let usize_ty = self.tcx.types.usize;
+        let bool_ty = self.tcx.types.bool;
         // bounds check:
         let len = self.temp(usize_ty, expr_span);
         let lt = self.temp(bool_ty, expr_span);
 
         // len = len(slice)
-        self.cfg.push_assign(block, source_info, len, Rvalue::Len(slice));
+        self.cfg.push_assign(
+            block,
+            source_info,
+            len,
+            Rvalue::Len(slice.into_place(self.tcx, self.typeck_results)),
+        );
         // lt = idx < len
         self.cfg.push_assign(
             block,
             source_info,
             lt,
-            Rvalue::BinaryOp(BinOp::Lt, Operand::Copy(Place::from(index)), Operand::Copy(len)),
+            Rvalue::BinaryOp(
+                BinOp::Lt,
+                box (Operand::Copy(Place::from(index)), Operand::Copy(len)),
+            ),
         );
         let msg = BoundsCheck { len: Operand::Move(len), index: Operand::Copy(Place::from(index)) };
         // assert!(lt, "...")
@@ -682,7 +695,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         expr_span: Span,
         source_info: SourceInfo,
     ) {
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
         let local = match base_place.base {
             PlaceBase::Local(local) => local,
             PlaceBase::Upvar { .. } => bug!("Expected PlacseBase::Local found Upvar"),
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index e602f4dd71d..822fbd91c94 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -8,6 +8,7 @@ use crate::build::{BlockAnd, BlockAndExtension, Builder};
 use crate::thir::*;
 use rustc_middle::middle::region;
 use rustc_middle::mir::AssertKind;
+use rustc_middle::mir::Place;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, UpvarSubsts};
 use rustc_span::Span;
@@ -19,33 +20,21 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// The operand returned from this function will *not be valid* after
     /// an ExprKind::Scope is passed, so please do *not* return it from
     /// functions to avoid bad miscompiles.
-    crate fn as_local_rvalue<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Rvalue<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
+    crate fn as_local_rvalue(
+        &mut self,
+        block: BasicBlock,
+        expr: &Expr<'_, 'tcx>,
+    ) -> BlockAnd<Rvalue<'tcx>> {
         let local_scope = self.local_scope();
         self.as_rvalue(block, Some(local_scope), expr)
     }
 
     /// Compile `expr`, yielding an rvalue.
-    fn as_rvalue<M>(
-        &mut self,
-        block: BasicBlock,
-        scope: Option<region::Scope>,
-        expr: M,
-    ) -> BlockAnd<Rvalue<'tcx>>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
-        let expr = self.hir.mirror(expr);
-        self.expr_as_rvalue(block, scope, expr)
-    }
-
-    fn expr_as_rvalue(
+    crate fn as_rvalue(
         &mut self,
         mut block: BasicBlock,
         scope: Option<region::Scope>,
-        expr: Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
     ) -> BlockAnd<Rvalue<'tcx>> {
         debug!("expr_as_rvalue(block={:?}, scope={:?}, expr={:?})", block, scope, expr);
 
@@ -71,8 +60,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::Unary { op, arg } => {
                 let arg = unpack!(block = this.as_operand(block, scope, arg));
                 // Check for -MIN on signed integers
-                if this.hir.check_overflow() && op == UnOp::Neg && expr.ty.is_signed() {
-                    let bool_ty = this.hir.bool_ty();
+                if this.check_overflow && op == UnOp::Neg && expr.ty.is_signed() {
+                    let bool_ty = this.tcx.types.bool;
 
                     let minval = this.minval_literal(expr_span, expr.ty);
                     let is_min = this.temp(bool_ty, expr_span);
@@ -81,7 +70,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         block,
                         source_info,
                         is_min,
-                        Rvalue::BinaryOp(BinOp::Eq, arg.to_copy(), minval),
+                        Rvalue::BinaryOp(BinOp::Eq, box (arg.to_copy(), minval)),
                     );
 
                     block = this.assert(
@@ -95,7 +84,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 block.and(Rvalue::UnaryOp(op, arg))
             }
             ExprKind::Box { value } => {
-                let value = this.hir.mirror(value);
                 // The `Box<T>` temporary created here is not a part of the HIR,
                 // and therefore is not considered during generator auto-trait
                 // determination. See the comment about `box` at `yield_in_scope`.
@@ -115,8 +103,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 // initialize the box contents:
                 unpack!(
-                    block =
-                        this.into(this.hir.tcx().mk_place_deref(Place::from(result)), block, value)
+                    block = this.expr_into_dest(
+                        this.tcx.mk_place_deref(Place::from(result)),
+                        block,
+                        value
+                    )
                 );
                 block.and(Rvalue::Use(Operand::Move(Place::from(result))))
             }
@@ -156,7 +147,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 //     to the same MIR as `let x = ();`.
 
                 // first process the set of fields
-                let el_ty = expr.ty.sequence_element_type(this.hir.tcx());
+                let el_ty = expr.ty.sequence_element_type(this.tcx);
                 let fields: Vec<_> = fields
                     .into_iter()
                     .map(|f| unpack!(block = this.as_operand(block, scope, f)))
@@ -174,12 +165,41 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 block.and(Rvalue::Aggregate(box AggregateKind::Tuple, fields))
             }
-            ExprKind::Closure { closure_id, substs, upvars, movability } => {
+            ExprKind::Closure { closure_id, substs, upvars, movability, ref fake_reads } => {
+                // Convert the closure fake reads, if any, from `ExprRef` to mir `Place`
+                // and push the fake reads.
+                // This must come before creating the operands. This is required in case
+                // there is a fake read and a borrow of the same path, since otherwise the
+                // fake read might interfere with the borrow. Consider an example like this
+                // one:
+                // ```
+                // let mut x = 0;
+                // let c = || {
+                //     &mut x; // mutable borrow of `x`
+                //     match x { _ => () } // fake read of `x`
+                // };
+                // ```
+                for (thir_place, cause, hir_id) in fake_reads.into_iter() {
+                    let place_builder = unpack!(block = this.as_place_builder(block, thir_place));
+
+                    if let Ok(place_builder_resolved) =
+                        place_builder.try_upvars_resolved(this.tcx, this.typeck_results)
+                    {
+                        let mir_place =
+                            place_builder_resolved.into_place(this.tcx, this.typeck_results);
+                        this.cfg.push_fake_read(
+                            block,
+                            this.source_info(this.tcx.hir().span(*hir_id)),
+                            *cause,
+                            mir_place,
+                        );
+                    }
+                }
+
                 // see (*) above
                 let operands: Vec<_> = upvars
                     .into_iter()
                     .map(|upvar| {
-                        let upvar = this.hir.mirror(upvar);
                         match Category::of(&upvar.kind) {
                             // Use as_place to avoid creating a temporary when
                             // moving a variable into a closure, so that
@@ -214,6 +234,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         }
                     })
                     .collect();
+
                 let result = match substs {
                     UpvarSubsts::Generator(substs) => {
                         // We implicitly set the discriminant to 0. See
@@ -230,7 +251,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 block.and(Rvalue::Use(Operand::Constant(box Constant {
                     span: expr_span,
                     user_ty: None,
-                    literal: ty::Const::zero_sized(this.hir.tcx(), this.hir.tcx().types.unit),
+                    literal: ty::Const::zero_sized(this.tcx, this.tcx.types.unit).into(),
                 })))
             }
             ExprKind::Yield { .. }
@@ -282,21 +303,21 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         rhs: Operand<'tcx>,
     ) -> BlockAnd<Rvalue<'tcx>> {
         let source_info = self.source_info(span);
-        let bool_ty = self.hir.bool_ty();
-        if self.hir.check_overflow() && op.is_checkable() && ty.is_integral() {
-            let result_tup = self.hir.tcx().intern_tup(&[ty, bool_ty]);
+        let bool_ty = self.tcx.types.bool;
+        if self.check_overflow && op.is_checkable() && ty.is_integral() {
+            let result_tup = self.tcx.intern_tup(&[ty, bool_ty]);
             let result_value = self.temp(result_tup, span);
 
             self.cfg.push_assign(
                 block,
                 source_info,
                 result_value,
-                Rvalue::CheckedBinaryOp(op, lhs.to_copy(), rhs.to_copy()),
+                Rvalue::CheckedBinaryOp(op, box (lhs.to_copy(), rhs.to_copy())),
             );
             let val_fld = Field::new(0);
             let of_fld = Field::new(1);
 
-            let tcx = self.hir.tcx();
+            let tcx = self.tcx;
             let val = tcx.mk_place_field(result_value, val_fld, ty);
             let of = tcx.mk_place_field(result_value, of_fld, bool_ty);
 
@@ -324,7 +345,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     block,
                     source_info,
                     is_zero,
-                    Rvalue::BinaryOp(BinOp::Eq, rhs.to_copy(), zero),
+                    Rvalue::BinaryOp(BinOp::Eq, box (rhs.to_copy(), zero)),
                 );
 
                 block = self.assert(block, Operand::Move(is_zero), false, zero_err, span);
@@ -345,13 +366,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         block,
                         source_info,
                         is_neg_1,
-                        Rvalue::BinaryOp(BinOp::Eq, rhs.to_copy(), neg_1),
+                        Rvalue::BinaryOp(BinOp::Eq, box (rhs.to_copy(), neg_1)),
                     );
                     self.cfg.push_assign(
                         block,
                         source_info,
                         is_min,
-                        Rvalue::BinaryOp(BinOp::Eq, lhs.to_copy(), min),
+                        Rvalue::BinaryOp(BinOp::Eq, box (lhs.to_copy(), min)),
                     );
 
                     let is_neg_1 = Operand::Move(is_neg_1);
@@ -360,14 +381,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         block,
                         source_info,
                         of,
-                        Rvalue::BinaryOp(BinOp::BitAnd, is_neg_1, is_min),
+                        Rvalue::BinaryOp(BinOp::BitAnd, box (is_neg_1, is_min)),
                     );
 
                     block = self.assert(block, Operand::Move(of), false, overflow_err, span);
                 }
             }
 
-            block.and(Rvalue::BinaryOp(op, lhs, rhs))
+            block.and(Rvalue::BinaryOp(op, box (lhs, rhs)))
         }
     }
 
@@ -377,7 +398,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         upvar_ty: Ty<'tcx>,
         temp_lifetime: Option<region::Scope>,
         mut block: BasicBlock,
-        arg: ExprRef<'tcx>,
+        arg: &Expr<'_, 'tcx>,
     ) -> BlockAnd<Operand<'tcx>> {
         let this = self;
 
@@ -398,7 +419,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // is same as that of the capture in the parent closure.
             PlaceBase::Upvar { .. } => {
                 let enclosing_upvars_resolved =
-                    arg_place_builder.clone().into_place(this.hir.tcx(), this.hir.typeck_results());
+                    arg_place_builder.clone().into_place(this.tcx, this.typeck_results);
 
                 match enclosing_upvars_resolved.as_ref() {
                     PlaceRef {
@@ -435,13 +456,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             Mutability::Mut => BorrowKind::Mut { allow_two_phase_borrow: false },
         };
 
-        let arg_place = arg_place_builder.into_place(this.hir.tcx(), this.hir.typeck_results());
+        let arg_place = arg_place_builder.into_place(this.tcx, this.typeck_results);
 
         this.cfg.push_assign(
             block,
             source_info,
             Place::from(temp),
-            Rvalue::Ref(this.hir.tcx().lifetimes.re_erased, borrow_kind, arg_place),
+            Rvalue::Ref(this.tcx.lifetimes.re_erased, borrow_kind, arg_place),
         );
 
         // See the comment in `expr_as_temp` and on the `rvalue_scopes` field for why
@@ -456,9 +477,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     // Helper to get a `-1` value of the appropriate type
     fn neg_1_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
         let param_ty = ty::ParamEnv::empty().and(ty);
-        let bits = self.hir.tcx().layout_of(param_ty).unwrap().size.bits();
+        let bits = self.tcx.layout_of(param_ty).unwrap().size.bits();
         let n = (!0u128) >> (128 - bits);
-        let literal = ty::Const::from_bits(self.hir.tcx(), n, param_ty);
+        let literal = ty::Const::from_bits(self.tcx, n, param_ty);
 
         self.literal_operand(span, literal)
     }
@@ -467,9 +488,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn minval_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
         assert!(ty.is_signed());
         let param_ty = ty::ParamEnv::empty().and(ty);
-        let bits = self.hir.tcx().layout_of(param_ty).unwrap().size.bits();
+        let bits = self.tcx.layout_of(param_ty).unwrap().size.bits();
         let n = 1 << (bits - 1);
-        let literal = ty::Const::from_bits(self.hir.tcx(), n, param_ty);
+        let literal = ty::Const::from_bits(self.tcx, n, param_ty);
 
         self.literal_operand(span, literal)
     }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_temp.rs b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
index 9984b527ffd..98b910ab21c 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_temp.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
@@ -4,40 +4,34 @@ use crate::build::scope::DropKind;
 use crate::build::{BlockAnd, BlockAndExtension, Builder};
 use crate::thir::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
-use rustc_hir as hir;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr` into a fresh temporary. This is used when building
     /// up rvalues so as to freeze the value that will be consumed.
-    crate fn as_temp<M>(
+    crate fn as_temp(
         &mut self,
         block: BasicBlock,
         temp_lifetime: Option<region::Scope>,
-        expr: M,
+        expr: &Expr<'_, 'tcx>,
         mutability: Mutability,
-    ) -> BlockAnd<Local>
-    where
-        M: Mirror<'tcx, Output = Expr<'tcx>>,
-    {
-        let expr = self.hir.mirror(expr);
-        //
+    ) -> BlockAnd<Local> {
         // this is the only place in mir building that we need to truly need to worry about
         // infinite recursion. Everything else does recurse, too, but it always gets broken up
         // at some point by inserting an intermediate temporary
-        ensure_sufficient_stack(|| self.expr_as_temp(block, temp_lifetime, expr, mutability))
+        ensure_sufficient_stack(|| self.as_temp_inner(block, temp_lifetime, expr, mutability))
     }
 
-    fn expr_as_temp(
+    fn as_temp_inner(
         &mut self,
         mut block: BasicBlock,
         temp_lifetime: Option<region::Scope>,
-        expr: Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
         mutability: Mutability,
     ) -> BlockAnd<Local> {
         debug!(
-            "expr_as_temp(block={:?}, temp_lifetime={:?}, expr={:?}, mutability={:?})",
+            "as_temp(block={:?}, temp_lifetime={:?}, expr={:?}, mutability={:?})",
             block, temp_lifetime, expr, mutability
         );
         let this = self;
@@ -65,13 +59,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             match expr.kind {
                 ExprKind::StaticRef { def_id, .. } => {
-                    assert!(!this.hir.tcx().is_thread_local_static(def_id));
+                    assert!(!this.tcx.is_thread_local_static(def_id));
                     local_decl.internal = true;
                     local_decl.local_info =
                         Some(box LocalInfo::StaticRef { def_id, is_thread_local: false });
                 }
                 ExprKind::ThreadLocalRef(def_id) => {
-                    assert!(this.hir.tcx().is_thread_local_static(def_id));
+                    assert!(this.tcx.is_thread_local_static(def_id));
                     local_decl.internal = true;
                     local_decl.local_info =
                         Some(box LocalInfo::StaticRef { def_id, is_thread_local: true });
@@ -89,7 +83,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // Don't bother with StorageLive and Dead for these temporaries,
             // they are never assigned.
             ExprKind::Break { .. } | ExprKind::Continue { .. } | ExprKind::Return { .. } => (),
-            ExprKind::Block { body: hir::Block { expr: None, targeted_by_break: false, .. } }
+            ExprKind::Block { body: Block { expr: None, targeted_by_break: false, .. } }
                 if expr_ty.is_never() => {}
             _ => {
                 this.cfg
@@ -114,7 +108,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
         }
 
-        unpack!(block = this.into(temp_place, block, expr));
+        unpack!(block = this.expr_into_dest(temp_place, block, expr));
 
         if let Some(temp_lifetime) = temp_lifetime {
             this.schedule_drop(expr_span, temp_lifetime, temp, DropKind::Value);
diff --git a/compiler/rustc_mir_build/src/build/expr/category.rs b/compiler/rustc_mir_build/src/build/expr/category.rs
index 9320b5810e3..0cadfa2f0a1 100644
--- a/compiler/rustc_mir_build/src/build/expr/category.rs
+++ b/compiler/rustc_mir_build/src/build/expr/category.rs
@@ -31,7 +31,7 @@ crate enum RvalueFunc {
 /// Determines the category for a given expression. Note that scope
 /// and paren expressions have no category.
 impl Category {
-    crate fn of(ek: &ExprKind<'_>) -> Option<Category> {
+    crate fn of(ek: &ExprKind<'_, '_>) -> Option<Category> {
         match *ek {
             ExprKind::Scope { .. } => None,
 
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index 235fe14cbf9..1e7ed3d95d2 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -7,19 +7,21 @@ use rustc_ast::InlineAsmOptions;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
+use rustc_index::vec::Idx;
 use rustc_middle::mir::*;
-use rustc_middle::ty::CanonicalUserTypeAnnotation;
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation};
+use std::iter;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, storing the result into `destination`, which
     /// is assumed to be uninitialized.
-    crate fn into_expr(
+    crate fn expr_into_dest(
         &mut self,
         destination: Place<'tcx>,
         mut block: BasicBlock,
-        expr: Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
     ) -> BlockAnd<()> {
-        debug!("into_expr(destination={:?}, block={:?}, expr={:?})", destination, block, expr);
+        debug!("expr_into_dest(destination={:?}, block={:?}, expr={:?})", destination, block, expr);
 
         // since we frequently have to reference `self` from within a
         // closure, where `self` would be shadowed, it's easier to
@@ -40,11 +42,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let region_scope = (region_scope, source_info);
                 ensure_sufficient_stack(|| {
                     this.in_scope(region_scope, lint_level, |this| {
-                        this.into(destination, block, value)
+                        this.expr_into_dest(destination, block, value)
                     })
                 })
             }
-            ExprKind::Block { body: ast_block } => {
+            ExprKind::Block { body: ref ast_block } => {
                 this.ast_block(destination, block, ast_block, source_info)
             }
             ExprKind::Match { scrutinee, arms } => {
@@ -58,17 +60,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 let mut then_block = this.cfg.start_new_block();
                 let mut else_block = this.cfg.start_new_block();
-                let term = TerminatorKind::if_(this.hir.tcx(), operand, then_block, else_block);
+                let term = TerminatorKind::if_(this.tcx, operand, then_block, else_block);
                 this.cfg.terminate(block, source_info, term);
 
-                unpack!(then_block = this.into(destination, then_block, then));
+                unpack!(then_block = this.expr_into_dest(destination, then_block, then));
                 else_block = if let Some(else_opt) = else_opt {
-                    unpack!(this.into(destination, else_block, else_opt))
+                    unpack!(this.expr_into_dest(destination, else_block, else_opt))
                 } else {
                     // Body of the `if` expression without an `else` clause must return `()`, thus
                     // we implicitly generate a `else {}` if it is not specified.
                     let correct_si = this.source_info(expr_span.shrink_to_hi());
-                    this.cfg.push_assign_unit(else_block, correct_si, destination, this.hir.tcx());
+                    this.cfg.push_assign_unit(else_block, correct_si, destination, this.tcx);
                     else_block
                 };
 
@@ -87,7 +89,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 join_block.unit()
             }
             ExprKind::NeverToAny { source } => {
-                let source = this.hir.mirror(source);
                 let is_call =
                     matches!(source.kind, ExprKind::Call { .. } | ExprKind::InlineAsm { .. });
 
@@ -110,18 +111,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::LogicalOp { op, lhs, rhs } => {
                 // And:
                 //
-                // [block: If(lhs)] -true-> [else_block: If(rhs)] -true-> [true_block]
-                //        |                          | (false)
-                //        +----------false-----------+------------------> [false_block]
+                // [block: If(lhs)] -true-> [else_block: dest = (rhs)]
+                //        | (false)
+                //  [shortcurcuit_block: dest = false]
                 //
                 // Or:
                 //
-                // [block: If(lhs)] -false-> [else_block: If(rhs)] -true-> [true_block]
-                //        | (true)                   | (false)
-                //  [true_block]               [false_block]
+                // [block: If(lhs)] -false-> [else_block: dest = (rhs)]
+                //        | (true)
+                //  [shortcurcuit_block: dest = true]
 
-                let (true_block, false_block, mut else_block, join_block) = (
-                    this.cfg.start_new_block(),
+                let (shortcircuit_block, mut else_block, join_block) = (
                     this.cfg.start_new_block(),
                     this.cfg.start_new_block(),
                     this.cfg.start_new_block(),
@@ -129,33 +129,31 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 let lhs = unpack!(block = this.as_local_operand(block, lhs));
                 let blocks = match op {
-                    LogicalOp::And => (else_block, false_block),
-                    LogicalOp::Or => (true_block, else_block),
+                    LogicalOp::And => (else_block, shortcircuit_block),
+                    LogicalOp::Or => (shortcircuit_block, else_block),
                 };
-                let term = TerminatorKind::if_(this.hir.tcx(), lhs, blocks.0, blocks.1);
+                let term = TerminatorKind::if_(this.tcx, lhs, blocks.0, blocks.1);
                 this.cfg.terminate(block, source_info, term);
 
-                let rhs = unpack!(else_block = this.as_local_operand(else_block, rhs));
-                let term = TerminatorKind::if_(this.hir.tcx(), rhs, true_block, false_block);
-                this.cfg.terminate(else_block, source_info, term);
-
                 this.cfg.push_assign_constant(
-                    true_block,
+                    shortcircuit_block,
                     source_info,
                     destination,
-                    Constant { span: expr_span, user_ty: None, literal: this.hir.true_literal() },
+                    Constant {
+                        span: expr_span,
+                        user_ty: None,
+                        literal: match op {
+                            LogicalOp::And => ty::Const::from_bool(this.tcx, false).into(),
+                            LogicalOp::Or => ty::Const::from_bool(this.tcx, true).into(),
+                        },
+                    },
                 );
+                this.cfg.goto(shortcircuit_block, source_info, join_block);
 
-                this.cfg.push_assign_constant(
-                    false_block,
-                    source_info,
-                    destination,
-                    Constant { span: expr_span, user_ty: None, literal: this.hir.false_literal() },
-                );
+                let rhs = unpack!(else_block = this.as_local_operand(else_block, rhs));
+                this.cfg.push_assign(else_block, source_info, destination, Rvalue::Use(rhs));
+                this.cfg.goto(else_block, source_info, join_block);
 
-                // Link up both branches:
-                this.cfg.goto(true_block, source_info, join_block);
-                this.cfg.goto(false_block, source_info, join_block);
                 join_block.unit()
             }
             ExprKind::Loop { body } => {
@@ -188,7 +186,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // introduce a unit temporary as the destination for the loop body.
                     let tmp = this.get_unit_temp();
                     // Execute the body, branching back to the test.
-                    let body_block_end = unpack!(this.into(tmp, body_block, body));
+                    let body_block_end = unpack!(this.expr_into_dest(tmp, body_block, body));
                     this.cfg.goto(body_block_end, source_info, loop_block);
 
                     // Loops are only exited by `break` expressions.
@@ -206,7 +204,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 this.record_operands_moved(&args);
 
-                debug!("into_expr: fn_span={:?}", fn_span);
+                debug!("expr_into_dest: fn_span={:?}", fn_span);
 
                 this.cfg.terminate(
                     block,
@@ -230,7 +228,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 this.diverge_from(block);
                 success.unit()
             }
-            ExprKind::Use { source } => this.into(destination, block, source),
+            ExprKind::Use { source } => this.expr_into_dest(destination, block, source),
             ExprKind::Borrow { arg, borrow_kind } => {
                 // We don't do this in `as_rvalue` because we use `as_place`
                 // for borrow expressions, so we cannot create an `RValue` that
@@ -241,8 +239,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     BorrowKind::Shared => unpack!(block = this.as_read_only_place(block, arg)),
                     _ => unpack!(block = this.as_place(block, arg)),
                 };
-                let borrow =
-                    Rvalue::Ref(this.hir.tcx().lifetimes.re_erased, borrow_kind, arg_place);
+                let borrow = Rvalue::Ref(this.tcx.lifetimes.re_erased, borrow_kind, arg_place);
                 this.cfg.push_assign(block, source_info, destination, borrow);
                 block.unit()
             }
@@ -255,7 +252,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 this.cfg.push_assign(block, source_info, destination, address_of);
                 block.unit()
             }
-            ExprKind::Adt { adt_def, variant_index, substs, user_ty, fields, base } => {
+            ExprKind::Adt { adt_def, variant_index, substs, user_ty, fields, ref base } => {
                 // See the notes for `ExprKind::Array` in `as_rvalue` and for
                 // `ExprKind::Borrow` above.
                 let is_union = adt_def.is_union();
@@ -270,7 +267,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     .map(|f| (f.name, unpack!(block = this.as_operand(block, Some(scope), f.expr))))
                     .collect();
 
-                let field_names = this.hir.all_fields(adt_def, variant_index);
+                let field_names: Vec<_> =
+                    (0..adt_def.variants[variant_index].fields.len()).map(Field::new).collect();
 
                 let fields: Vec<_> = if let Some(FruInfo { base, field_types }) = base {
                     let place_builder = unpack!(block = this.as_place_builder(block, base));
@@ -278,9 +276,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // MIR does not natively support FRU, so for each
                     // base-supplied field, generate an operand that
                     // reads it from the base.
-                    field_names
-                        .into_iter()
-                        .zip(field_types.into_iter())
+                    iter::zip(field_names, *field_types)
                         .map(|(n, ty)| match fields_map.get(&n) {
                             Some(v) => v.clone(),
                             None => {
@@ -288,7 +284,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 this.consume_by_copy_or_move(
                                     place_builder
                                         .field(n, ty)
-                                        .into_place(this.hir.tcx(), this.hir.typeck_results()),
+                                        .into_place(this.tcx, this.typeck_results),
                                 )
                             }
                         })
@@ -325,7 +321,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 use rustc_middle::mir;
                 let operands = operands
                     .into_iter()
-                    .map(|op| match op {
+                    .map(|op| match *op {
                         thir::InlineAsmOperand::In { reg, expr } => mir::InlineAsmOperand::In {
                             reg,
                             value: unpack!(block = this.as_local_operand(block, expr)),
@@ -334,7 +330,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             mir::InlineAsmOperand::Out {
                                 reg,
                                 late,
-                                place: expr.map(|expr| unpack!(block = this.as_place(block, expr))),
+                                place: expr
+                                    .as_ref()
+                                    .map(|expr| unpack!(block = this.as_place(block, expr))),
                             }
                         }
                         thir::InlineAsmOperand::InOut { reg, late, expr } => {
@@ -352,14 +350,16 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 reg,
                                 late,
                                 in_value: unpack!(block = this.as_local_operand(block, in_expr)),
-                                out_place: out_expr.map(|out_expr| {
+                                out_place: out_expr.as_ref().map(|out_expr| {
                                     unpack!(block = this.as_place(block, out_expr))
                                 }),
                             }
                         }
-                        thir::InlineAsmOperand::Const { expr } => mir::InlineAsmOperand::Const {
-                            value: unpack!(block = this.as_local_operand(block, expr)),
-                        },
+                        thir::InlineAsmOperand::Const { value, span } => {
+                            mir::InlineAsmOperand::Const {
+                                value: box Constant { span, user_ty: None, literal: value.into() },
+                            }
+                        }
                         thir::InlineAsmOperand::SymFn { expr } => {
                             mir::InlineAsmOperand::SymFn { value: box this.as_constant(expr) }
                         }
@@ -394,7 +394,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::AssignOp { .. }
             | ExprKind::LlvmInlineAsm { .. } => {
                 unpack!(block = this.stmt_expr(block, expr, None));
-                this.cfg.push_assign_unit(block, source_info, destination, this.hir.tcx());
+                this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
                 block.unit()
             }
 
@@ -417,7 +417,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 block.unit()
             }
             ExprKind::Index { .. } | ExprKind::Deref { .. } | ExprKind::Field { .. } => {
-                debug_assert!(Category::of(&expr.kind) == Some(Category::Place));
+                debug_assert_eq!(Category::of(&expr.kind), Some(Category::Place));
 
                 // Create a "fake" temporary variable so that we check that the
                 // value is Sized. Usually, this is caught in type checking, but
@@ -426,8 +426,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     this.local_decls.push(LocalDecl::new(expr.ty, expr.span));
                 }
 
-                debug_assert!(Category::of(&expr.kind) == Some(Category::Place));
-
                 let place = unpack!(block = this.as_place(block, expr));
                 let rvalue = Rvalue::Use(this.consume_by_copy_or_move(place));
                 this.cfg.push_assign(block, source_info, destination, rvalue);
diff --git a/compiler/rustc_mir_build/src/build/expr/mod.rs b/compiler/rustc_mir_build/src/build/expr/mod.rs
index ac8c7e725e1..539de80cab7 100644
--- a/compiler/rustc_mir_build/src/build/expr/mod.rs
+++ b/compiler/rustc_mir_build/src/build/expr/mod.rs
@@ -9,7 +9,7 @@
 //! a type that is not `Copy`, then using any of these functions will
 //! "move" the value out of its current home (if any).
 //!
-//! - `into` -- writes the value into a specific location, which
+//! - `expr_into_dest` -- writes the value into a specific location, which
 //!   should be uninitialized
 //! - `as_operand` -- evaluates the value and yields an `Operand`,
 //!   suitable for use as an argument to an `Rvalue`
@@ -62,7 +62,7 @@
 
 mod as_constant;
 mod as_operand;
-mod as_place;
+pub mod as_place;
 mod as_rvalue;
 mod as_temp;
 mod category;
diff --git a/compiler/rustc_mir_build/src/build/expr/stmt.rs b/compiler/rustc_mir_build/src/build/expr/stmt.rs
index f117689d940..f01315fc5db 100644
--- a/compiler/rustc_mir_build/src/build/expr/stmt.rs
+++ b/compiler/rustc_mir_build/src/build/expr/stmt.rs
@@ -13,7 +13,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     crate fn stmt_expr(
         &mut self,
         mut block: BasicBlock,
-        expr: Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
         statement_scope: Option<region::Scope>,
     ) -> BlockAnd<()> {
         let this = self;
@@ -21,29 +21,25 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let source_info = this.source_info(expr.span);
         // Handle a number of expressions that don't need a destination at all. This
         // avoids needing a mountain of temporary `()` variables.
-        let expr2 = expr.clone();
         match expr.kind {
             ExprKind::Scope { region_scope, lint_level, value } => {
-                let value = this.hir.mirror(value);
                 this.in_scope((region_scope, source_info), lint_level, |this| {
                     this.stmt_expr(block, value, statement_scope)
                 })
             }
             ExprKind::Assign { lhs, rhs } => {
-                let lhs = this.hir.mirror(lhs);
-                let rhs = this.hir.mirror(rhs);
                 let lhs_span = lhs.span;
 
                 // Note: we evaluate assignments right-to-left. This
                 // is better for borrowck interaction with overloaded
                 // operators like x[j] = x[i].
 
-                debug!("stmt_expr Assign block_context.push(SubExpr) : {:?}", expr2);
+                debug!("stmt_expr Assign block_context.push(SubExpr) : {:?}", expr);
                 this.block_context.push(BlockFrame::SubExpr);
 
                 // Generate better code for things that don't need to be
                 // dropped.
-                if this.hir.needs_drop(lhs.ty) {
+                if lhs.ty.needs_drop(this.tcx, this.param_env) {
                     let rhs = unpack!(block = this.as_local_operand(block, rhs));
                     let lhs = unpack!(block = this.as_place(block, lhs));
                     unpack!(block = this.build_drop_and_replace(block, lhs_span, lhs, rhs));
@@ -65,10 +61,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // only affects weird things like `x += {x += 1; x}`
                 // -- is that equal to `x + (x + 1)` or `2*(x+1)`?
 
-                let lhs = this.hir.mirror(lhs);
                 let lhs_ty = lhs.ty;
 
-                debug!("stmt_expr AssignOp block_context.push(SubExpr) : {:?}", expr2);
+                debug!("stmt_expr AssignOp block_context.push(SubExpr) : {:?}", expr);
                 this.block_context.push(BlockFrame::SubExpr);
 
                 // As above, RTL.
@@ -90,24 +85,27 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::Continue { label } => {
                 this.break_scope(block, None, BreakableTarget::Continue(label), source_info)
             }
-            ExprKind::Break { label, value } => {
-                this.break_scope(block, value, BreakableTarget::Break(label), source_info)
-            }
+            ExprKind::Break { label, value } => this.break_scope(
+                block,
+                value.as_deref(),
+                BreakableTarget::Break(label),
+                source_info,
+            ),
             ExprKind::Return { value } => {
-                this.break_scope(block, value, BreakableTarget::Return, source_info)
+                this.break_scope(block, value.as_deref(), BreakableTarget::Return, source_info)
             }
             ExprKind::LlvmInlineAsm { asm, outputs, inputs } => {
-                debug!("stmt_expr LlvmInlineAsm block_context.push(SubExpr) : {:?}", expr2);
+                debug!("stmt_expr LlvmInlineAsm block_context.push(SubExpr) : {:?}", expr);
                 this.block_context.push(BlockFrame::SubExpr);
                 let outputs = outputs
                     .into_iter()
-                    .map(|output| unpack!(block = this.as_place(block, output)))
+                    .map(|output| unpack!(block = this.as_place(block, &output)))
                     .collect::<Vec<_>>()
                     .into_boxed_slice();
                 let inputs = inputs
                     .into_iter()
                     .map(|input| {
-                        (input.span(), unpack!(block = this.as_local_operand(block, input)))
+                        (input.span, unpack!(block = this.as_local_operand(block, &input)))
                     })
                     .collect::<Vec<_>>()
                     .into_boxed_slice();
@@ -140,15 +138,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // it is usually better to focus on `the_value` rather
                 // than the entirety of block(s) surrounding it.
                 let adjusted_span = (|| {
-                    if let ExprKind::Block { body } = expr.kind {
+                    if let ExprKind::Block { body } = &expr.kind {
                         if let Some(tail_expr) = &body.expr {
-                            let mut expr = tail_expr;
-                            while let rustc_hir::ExprKind::Block(subblock, _label) = &expr.kind {
-                                if let Some(subtail_expr) = &subblock.expr {
-                                    expr = subtail_expr
-                                } else {
-                                    break;
-                                }
+                            let mut expr = &*tail_expr;
+                            while let ExprKind::Block {
+                                body: Block { expr: Some(nested_expr), .. },
+                            }
+                            | ExprKind::Scope { value: nested_expr, .. } = &expr.kind
+                            {
+                                expr = nested_expr;
                             }
                             this.block_context.push(BlockFrame::TailExpr {
                                 tail_result_is_ignored: true,
diff --git a/compiler/rustc_mir_build/src/build/into.rs b/compiler/rustc_mir_build/src/build/into.rs
deleted file mode 100644
index 7264e495b84..00000000000
--- a/compiler/rustc_mir_build/src/build/into.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-//! In general, there are a number of things for which it's convenient
-//! to just call `builder.into` and have it emit its result into a
-//! given location. This is basically for expressions or things that can be
-//! wrapped up as expressions (e.g., blocks). To make this ergonomic, we use this
-//! latter `EvalInto` trait.
-
-use crate::build::{BlockAnd, Builder};
-use crate::thir::*;
-use rustc_middle::mir::*;
-
-pub(in crate::build) trait EvalInto<'tcx> {
-    fn eval_into(
-        self,
-        builder: &mut Builder<'_, 'tcx>,
-        destination: Place<'tcx>,
-        block: BasicBlock,
-    ) -> BlockAnd<()>;
-}
-
-impl<'a, 'tcx> Builder<'a, 'tcx> {
-    crate fn into<E>(
-        &mut self,
-        destination: Place<'tcx>,
-        block: BasicBlock,
-        expr: E,
-    ) -> BlockAnd<()>
-    where
-        E: EvalInto<'tcx>,
-    {
-        expr.eval_into(self, destination, block)
-    }
-}
-
-impl<'tcx> EvalInto<'tcx> for ExprRef<'tcx> {
-    fn eval_into(
-        self,
-        builder: &mut Builder<'_, 'tcx>,
-        destination: Place<'tcx>,
-        block: BasicBlock,
-    ) -> BlockAnd<()> {
-        let expr = builder.hir.mirror(self);
-        builder.into_expr(destination, block, expr)
-    }
-}
-
-impl<'tcx> EvalInto<'tcx> for Expr<'tcx> {
-    fn eval_into(
-        self,
-        builder: &mut Builder<'_, 'tcx>,
-        destination: Place<'tcx>,
-        block: BasicBlock,
-    ) -> BlockAnd<()> {
-        builder.into_expr(destination, block, self)
-    }
-}
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index fde007ec011..0e422dc3c63 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -5,6 +5,7 @@
 //! This also includes code for pattern bindings in `let` statements and
 //! function parameters.
 
+use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::scope::DropKind;
 use crate::build::ForGuard::{self, OutsideGuard, RefWithinGuard};
 use crate::build::{BlockAnd, BlockAndExtension, Builder};
@@ -89,14 +90,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         destination: Place<'tcx>,
         span: Span,
         mut block: BasicBlock,
-        scrutinee: ExprRef<'tcx>,
-        arms: Vec<Arm<'tcx>>,
+        scrutinee: &Expr<'_, 'tcx>,
+        arms: &[Arm<'_, 'tcx>],
     ) -> BlockAnd<()> {
-        let scrutinee_span = scrutinee.span();
+        let scrutinee_span = scrutinee.span;
         let scrutinee_place =
             unpack!(block = self.lower_scrutinee(block, scrutinee, scrutinee_span,));
 
-        let mut arm_candidates = self.create_match_candidates(scrutinee_place, &arms);
+        let mut arm_candidates = self.create_match_candidates(scrutinee_place.clone(), &arms);
 
         let match_has_guard = arms.iter().any(|arm| arm.guard.is_some());
         let mut candidates =
@@ -119,10 +120,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn lower_scrutinee(
         &mut self,
         mut block: BasicBlock,
-        scrutinee: ExprRef<'tcx>,
+        scrutinee: &Expr<'_, 'tcx>,
         scrutinee_span: Span,
-    ) -> BlockAnd<Place<'tcx>> {
-        let scrutinee_place = unpack!(block = self.as_place(block, scrutinee));
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        let scrutinee_place_builder = unpack!(block = self.as_place_builder(block, scrutinee));
         // Matching on a `scrutinee_place` with an uninhabited type doesn't
         // generate any memory reads by itself, and so if the place "expression"
         // contains unsafe operations like raw pointer dereferences or union
@@ -138,25 +139,31 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // uninhabited value. If we get never patterns, those will check that
         // the place is initialized, and so this read would only be used to
         // check safety.
-        let cause_matched_place = FakeReadCause::ForMatchedPlace;
+        let cause_matched_place = FakeReadCause::ForMatchedPlace(None);
         let source_info = self.source_info(scrutinee_span);
-        self.cfg.push_fake_read(block, source_info, cause_matched_place, scrutinee_place);
 
-        block.and(scrutinee_place)
+        if let Ok(scrutinee_builder) =
+            scrutinee_place_builder.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+        {
+            let scrutinee_place = scrutinee_builder.into_place(self.tcx, self.typeck_results);
+            self.cfg.push_fake_read(block, source_info, cause_matched_place, scrutinee_place);
+        }
+
+        block.and(scrutinee_place_builder)
     }
 
     /// Create the initial `Candidate`s for a `match` expression.
     fn create_match_candidates<'pat>(
         &mut self,
-        scrutinee: Place<'tcx>,
-        arms: &'pat [Arm<'tcx>],
-    ) -> Vec<(&'pat Arm<'tcx>, Candidate<'pat, 'tcx>)> {
+        scrutinee: PlaceBuilder<'tcx>,
+        arms: &'pat [Arm<'pat, 'tcx>],
+    ) -> Vec<(&'pat Arm<'pat, 'tcx>, Candidate<'pat, 'tcx>)> {
         // Assemble a list of candidates: there is one candidate per pattern,
         // which means there may be more than one candidate *per arm*.
         arms.iter()
             .map(|arm| {
                 let arm_has_guard = arm.guard.is_some();
-                let arm_candidate = Candidate::new(scrutinee, &arm.pattern, arm_has_guard);
+                let arm_candidate = Candidate::new(scrutinee.clone(), &arm.pattern, arm_has_guard);
                 (arm, arm_candidate)
             })
             .collect()
@@ -222,9 +229,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn lower_match_arms(
         &mut self,
         destination: Place<'tcx>,
-        scrutinee_place: Place<'tcx>,
+        scrutinee_place_builder: PlaceBuilder<'tcx>,
         scrutinee_span: Span,
-        arm_candidates: Vec<(&'_ Arm<'tcx>, Candidate<'_, 'tcx>)>,
+        arm_candidates: Vec<(&'_ Arm<'_, 'tcx>, Candidate<'_, 'tcx>)>,
         outer_source_info: SourceInfo,
         fake_borrow_temps: Vec<(Place<'tcx>, Local)>,
     ) -> BlockAnd<()> {
@@ -236,13 +243,33 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let arm_source_info = self.source_info(arm.span);
                 let arm_scope = (arm.scope, arm_source_info);
                 self.in_scope(arm_scope, arm.lint_level, |this| {
-                    let body = this.hir.mirror(arm.body.clone());
+                    // `try_upvars_resolved` may fail if it is unable to resolve the given
+                    // `PlaceBuilder` inside a closure. In this case, we don't want to include
+                    // a scrutinee place. `scrutinee_place_builder` will fail to be resolved
+                    // if the only match arm is a wildcard (`_`).
+                    // Example:
+                    // ```
+                    // let foo = (0, 1);
+                    // let c = || {
+                    //    match foo { _ => () };
+                    // };
+                    // ```
+                    let mut opt_scrutinee_place: Option<(Option<&Place<'tcx>>, Span)> = None;
+                    let scrutinee_place: Place<'tcx>;
+                    if let Ok(scrutinee_builder) = scrutinee_place_builder
+                        .clone()
+                        .try_upvars_resolved(this.tcx, this.typeck_results)
+                    {
+                        scrutinee_place =
+                            scrutinee_builder.into_place(this.tcx, this.typeck_results);
+                        opt_scrutinee_place = Some((Some(&scrutinee_place), scrutinee_span));
+                    }
                     let scope = this.declare_bindings(
                         None,
                         arm.span,
                         &arm.pattern,
                         ArmHasGuard(arm.guard.is_some()),
-                        Some((Some(&scrutinee_place), scrutinee_span)),
+                        opt_scrutinee_place,
                     );
 
                     let arm_block = this.bind_pattern(
@@ -259,7 +286,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         this.source_scope = source_scope;
                     }
 
-                    this.into(destination, arm_block, body)
+                    this.expr_into_dest(destination, arm_block, &arm.body)
                 })
             })
             .collect();
@@ -286,7 +313,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         outer_source_info: SourceInfo,
         candidate: Candidate<'_, 'tcx>,
-        guard: Option<&Guard<'tcx>>,
+        guard: Option<&Guard<'_, 'tcx>>,
         fake_borrow_temps: &Vec<(Place<'tcx>, Local)>,
         scrutinee_span: Span,
         arm_span: Option<Span>,
@@ -362,18 +389,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         mut block: BasicBlock,
         irrefutable_pat: Pat<'tcx>,
-        initializer: ExprRef<'tcx>,
+        initializer: &Expr<'_, 'tcx>,
     ) -> BlockAnd<()> {
         match *irrefutable_pat.kind {
             // Optimize the case of `let x = ...` to write directly into `x`
             PatKind::Binding { mode: BindingMode::ByValue, var, subpattern: None, .. } => {
                 let place =
                     self.storage_live_binding(block, var, irrefutable_pat.span, OutsideGuard, true);
-                unpack!(block = self.into(place, block, initializer));
+                unpack!(block = self.expr_into_dest(place, block, initializer));
 
                 // Inject a fake read, see comments on `FakeReadCause::ForLet`.
                 let source_info = self.source_info(irrefutable_pat.span);
-                self.cfg.push_fake_read(block, source_info, FakeReadCause::ForLet, place);
+                self.cfg.push_fake_read(block, source_info, FakeReadCause::ForLet(None), place);
 
                 self.schedule_drop_for_binding(var, irrefutable_pat.span, OutsideGuard);
                 block.unit()
@@ -404,17 +431,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             } => {
                 let place =
                     self.storage_live_binding(block, var, irrefutable_pat.span, OutsideGuard, true);
-                unpack!(block = self.into(place, block, initializer));
+                unpack!(block = self.expr_into_dest(place, block, initializer));
 
                 // Inject a fake read, see comments on `FakeReadCause::ForLet`.
                 let pattern_source_info = self.source_info(irrefutable_pat.span);
-                let cause_let = FakeReadCause::ForLet;
+                let cause_let = FakeReadCause::ForLet(None);
                 self.cfg.push_fake_read(block, pattern_source_info, cause_let, place);
 
                 let ty_source_info = self.source_info(user_ty_span);
                 let user_ty = pat_ascription_ty.user_ty(
                     &mut self.canonical_user_type_annotations,
-                    place.ty(&self.local_decls, self.hir.tcx()).ty,
+                    place.ty(&self.local_decls, self.tcx).ty,
                     ty_source_info.span,
                 );
                 self.cfg.push(
@@ -447,8 +474,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             _ => {
-                let place = unpack!(block = self.as_place(block, initializer));
-                self.place_into_pattern(block, irrefutable_pat, place, true)
+                let place_builder = unpack!(block = self.as_place_builder(block, initializer));
+                self.place_into_pattern(block, irrefutable_pat, place_builder, true)
             }
         }
     }
@@ -457,14 +484,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         block: BasicBlock,
         irrefutable_pat: Pat<'tcx>,
-        initializer: Place<'tcx>,
+        initializer: PlaceBuilder<'tcx>,
         set_match_place: bool,
     ) -> BlockAnd<()> {
-        let mut candidate = Candidate::new(initializer, &irrefutable_pat, false);
-
+        let mut candidate = Candidate::new(initializer.clone(), &irrefutable_pat, false);
         let fake_borrow_temps =
             self.lower_match_tree(block, irrefutable_pat.span, false, &mut [&mut candidate]);
-
         // For matches and function arguments, the place that is being matched
         // can be set when creating the variables. But the place for
         // let PATTERN = ... might not even exist until we do the assignment.
@@ -479,7 +504,27 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         VarBindingForm { opt_match_place: Some((ref mut match_place, _)), .. },
                     )))) = self.local_decls[local].local_info
                     {
-                        *match_place = Some(initializer);
+                        // `try_upvars_resolved` may fail if it is unable to resolve the given
+                        // `PlaceBuilder` inside a closure. In this case, we don't want to include
+                        // a scrutinee place. `scrutinee_place_builder` will fail for destructured
+                        // assignments. This is because a closure only captures the precise places
+                        // that it will read and as a result a closure may not capture the entire
+                        // tuple/struct and rather have individual places that will be read in the
+                        // final MIR.
+                        // Example:
+                        // ```
+                        // let foo = (0, 1);
+                        // let c = || {
+                        //    let (v1, v2) = foo;
+                        // };
+                        // ```
+                        if let Ok(match_pair_resolved) =
+                            initializer.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+                        {
+                            let place =
+                                match_pair_resolved.into_place(self.tcx, self.typeck_results);
+                            *match_place = Some(place);
+                        }
                     } else {
                         bug!("Let binding to non-user variable.")
                     }
@@ -556,7 +601,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let local_id = self.var_local_id(var, for_guard);
         let source_info = self.source_info(span);
         self.cfg.push(block, Statement { source_info, kind: StatementKind::StorageLive(local_id) });
-        let region_scope = self.hir.region_scope_tree.var_scope(var.local_id);
+        let region_scope = self.region_scope_tree.var_scope(var.local_id);
         if schedule_drop {
             self.schedule_drop(span, region_scope, local_id, DropKind::Storage);
         }
@@ -565,7 +610,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     crate fn schedule_drop_for_binding(&mut self, var: HirId, span: Span, for_guard: ForGuard) {
         let local_id = self.var_local_id(var, for_guard);
-        let region_scope = self.hir.region_scope_tree.var_scope(var.local_id);
+        let region_scope = self.region_scope_tree.var_scope(var.local_id);
         self.schedule_drop(span, region_scope, local_id, DropKind::Value);
     }
 
@@ -718,7 +763,7 @@ struct Candidate<'pat, 'tcx> {
 }
 
 impl<'tcx, 'pat> Candidate<'pat, 'tcx> {
-    fn new(place: Place<'tcx>, pattern: &'pat Pat<'tcx>, has_guard: bool) -> Self {
+    fn new(place: PlaceBuilder<'tcx>, pattern: &'pat Pat<'tcx>, has_guard: bool) -> Self {
         Candidate {
             span: pattern.span,
             has_guard,
@@ -792,7 +837,7 @@ struct Ascription<'tcx> {
 #[derive(Clone, Debug)]
 crate struct MatchPair<'pat, 'tcx> {
     // this place...
-    place: Place<'tcx>,
+    place: PlaceBuilder<'tcx>,
 
     // ... must match this pattern.
     pattern: &'pat Pat<'tcx>,
@@ -1071,7 +1116,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                     fake_borrows.insert(Place {
                         local: source.local,
-                        projection: self.hir.tcx().intern_place_elems(proj_base),
+                        projection: self.tcx.intern_place_elems(proj_base),
                     });
                 }
             }
@@ -1199,7 +1244,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         &mut otherwise,
                         pats,
                         or_span,
-                        place,
+                        place.clone(),
                         fake_borrows,
                     );
                 });
@@ -1225,12 +1270,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         otherwise: &mut Option<BasicBlock>,
         pats: &'pat [Pat<'tcx>],
         or_span: Span,
-        place: Place<'tcx>,
+        place: PlaceBuilder<'tcx>,
         fake_borrows: &mut Option<FxHashSet<Place<'tcx>>>,
     ) {
         debug!("test_or_pattern:\ncandidate={:#?}\npats={:#?}", candidate, pats);
-        let mut or_candidates: Vec<_> =
-            pats.iter().map(|pat| Candidate::new(place, pat, candidate.has_guard)).collect();
+        let mut or_candidates: Vec<_> = pats
+            .iter()
+            .map(|pat| Candidate::new(place.clone(), pat, candidate.has_guard))
+            .collect();
         let mut or_candidate_refs: Vec<_> = or_candidates.iter_mut().collect();
         let otherwise = if candidate.otherwise_block.is_some() {
             &mut candidate.otherwise_block
@@ -1413,7 +1460,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // extract the match-pair from the highest priority candidate
         let match_pair = &candidates.first().unwrap().match_pairs[0];
         let mut test = self.test(match_pair);
-        let match_place = match_pair.place;
+        let match_place = match_pair.place.clone();
 
         // most of the time, the test to perform is simply a function
         // of the main candidate; but for a test like SwitchInt, we
@@ -1439,7 +1486,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         // Insert a Shallow borrow of any places that is switched on.
         if let Some(fb) = fake_borrows {
-            fb.insert(match_place);
+            if let Ok(match_place_resolved) =
+                match_place.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+            {
+                let resolved_place = match_place_resolved.into_place(self.tcx, self.typeck_results);
+                fb.insert(resolved_place);
+            }
         }
 
         // perform the test, branching to one of N blocks. For each of
@@ -1457,7 +1509,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // encounter a candidate where the test is not relevant; at
         // that point, we stop sorting.
         while let Some(candidate) = candidates.first_mut() {
-            if let Some(idx) = self.sort_candidate(&match_place, &test, candidate) {
+            if let Some(idx) = self.sort_candidate(&match_place.clone(), &test, candidate) {
                 let (candidate, rest) = candidates.split_first_mut().unwrap();
                 target_candidates[idx].push(candidate);
                 candidates = rest;
@@ -1550,7 +1602,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         fake_borrows: &'b FxHashSet<Place<'tcx>>,
         temp_span: Span,
     ) -> Vec<(Place<'tcx>, Local)> {
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
 
         debug!("add_fake_borrows fake_borrows = {:?}", fake_borrows);
 
@@ -1613,7 +1665,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         candidate: Candidate<'pat, 'tcx>,
         parent_bindings: &[(Vec<Binding<'tcx>>, Vec<Ascription<'tcx>>)],
-        guard: Option<&Guard<'tcx>>,
+        guard: Option<&Guard<'_, 'tcx>>,
         fake_borrows: &Vec<(Place<'tcx>, Local)>,
         scrutinee_span: Span,
         arm_span: Option<Span>,
@@ -1727,7 +1779,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         //    * So we eagerly create the reference for the arm and then take a
         //      reference to that.
         if let Some(guard) = guard {
-            let tcx = self.hir.tcx();
+            let tcx = self.tcx;
             let bindings = parent_bindings
                 .iter()
                 .flat_map(|(bindings, _)| bindings)
@@ -1749,37 +1801,46 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             let (guard_span, (post_guard_block, otherwise_post_guard_block)) = match guard {
                 Guard::If(e) => {
-                    let e = self.hir.mirror(e.clone());
                     let source_info = self.source_info(e.span);
                     (e.span, self.test_bool(block, e, source_info))
                 }
                 Guard::IfLet(pat, scrutinee) => {
-                    let scrutinee_span = scrutinee.span();
-                    let scrutinee_place = unpack!(
-                        block = self.lower_scrutinee(block, scrutinee.clone(), scrutinee_span)
-                    );
-                    let mut guard_candidate = Candidate::new(scrutinee_place, &pat, false);
+                    let scrutinee_span = scrutinee.span;
+                    let scrutinee_place_builder =
+                        unpack!(block = self.lower_scrutinee(block, scrutinee, scrutinee_span));
+                    let mut guard_candidate =
+                        Candidate::new(scrutinee_place_builder.clone(), &pat, false);
                     let wildcard = Pat::wildcard_from_ty(pat.ty);
-                    let mut otherwise_candidate = Candidate::new(scrutinee_place, &wildcard, false);
+                    let mut otherwise_candidate =
+                        Candidate::new(scrutinee_place_builder.clone(), &wildcard, false);
                     let fake_borrow_temps = self.lower_match_tree(
                         block,
                         pat.span,
                         false,
                         &mut [&mut guard_candidate, &mut otherwise_candidate],
                     );
+                    let mut opt_scrutinee_place: Option<(Option<&Place<'tcx>>, Span)> = None;
+                    let scrutinee_place: Place<'tcx>;
+                    if let Ok(scrutinee_builder) =
+                        scrutinee_place_builder.try_upvars_resolved(self.tcx, self.typeck_results)
+                    {
+                        scrutinee_place =
+                            scrutinee_builder.into_place(self.tcx, self.typeck_results);
+                        opt_scrutinee_place = Some((Some(&scrutinee_place), scrutinee_span));
+                    }
                     self.declare_bindings(
                         None,
                         pat.span.to(arm_span.unwrap()),
                         pat,
                         ArmHasGuard(false),
-                        Some((Some(&scrutinee_place), scrutinee.span())),
+                        opt_scrutinee_place,
                     );
                     let post_guard_block = self.bind_pattern(
                         self.source_info(pat.span),
                         guard_candidate,
                         None,
                         &fake_borrow_temps,
-                        scrutinee.span(),
+                        scrutinee.span,
                         None,
                         None,
                     );
@@ -1888,7 +1949,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             let user_ty = ascription.user_ty.clone().user_ty(
                 &mut self.canonical_user_type_annotations,
-                ascription.source.ty(&self.local_decls, self.hir.tcx()).ty,
+                ascription.source.ty(&self.local_decls, self.tcx).ty,
                 source_info.span,
             );
             self.cfg.push(
@@ -1917,7 +1978,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // Assign each of the bindings. Since we are binding for a
         // guard expression, this will never trigger moves out of the
         // candidate.
-        let re_erased = self.hir.tcx().lifetimes.re_erased;
+        let re_erased = self.tcx.lifetimes.re_erased;
         for binding in bindings {
             debug!("bind_matched_candidate_for_guard(binding={:?})", binding);
             let source_info = self.source_info(binding.span);
@@ -1966,7 +2027,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     {
         debug!("bind_matched_candidate_for_arm_body(block={:?})", block);
 
-        let re_erased = self.hir.tcx().lifetimes.re_erased;
+        let re_erased = self.tcx.lifetimes.re_erased;
         // Assign each of the bindings. This may trigger moves out of the candidate.
         for binding in bindings {
             let source_info = self.source_info(binding.span);
@@ -2015,7 +2076,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             var_id, name, mode, var_ty, visibility_scope, source_info
         );
 
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
         let debug_source_info = SourceInfo { span: source_info.span, scope: visibility_scope };
         let binding_mode = match mode {
             BindingMode::ByValue => ty::BindingMode::BindByValue(mutability),
diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs
index ddfaeafc07c..3ad143a57ff 100644
--- a/compiler/rustc_mir_build/src/build/matches/simplify.rs
+++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs
@@ -12,11 +12,11 @@
 //! sort of test: for example, testing which variant an enum is, or
 //! testing a value against a constant.
 
+use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::{Ascription, Binding, Candidate, MatchPair};
 use crate::build::Builder;
 use crate::thir::{self, *};
 use rustc_hir::RangeEnd;
-use rustc_middle::mir::Place;
 use rustc_middle::ty;
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_target::abi::{Integer, Size};
@@ -68,11 +68,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             let match_pairs = mem::take(&mut candidate.match_pairs);
 
             if let [MatchPair { pattern: Pat { kind: box PatKind::Or { pats }, .. }, place }] =
-                *match_pairs
+                &*match_pairs
             {
                 existing_bindings.extend_from_slice(&new_bindings);
                 mem::swap(&mut candidate.bindings, &mut existing_bindings);
-                candidate.subcandidates = self.create_or_subcandidates(candidate, place, pats);
+                candidate.subcandidates =
+                    self.create_or_subcandidates(candidate, place.clone(), pats);
                 return true;
             }
 
@@ -125,12 +126,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn create_or_subcandidates<'pat>(
         &mut self,
         candidate: &Candidate<'pat, 'tcx>,
-        place: Place<'tcx>,
+        place: PlaceBuilder<'tcx>,
         pats: &'pat [Pat<'tcx>],
     ) -> Vec<Candidate<'pat, 'tcx>> {
         pats.iter()
             .map(|pat| {
-                let mut candidate = Candidate::new(place, pat, candidate.has_guard);
+                let mut candidate = Candidate::new(place.clone(), pat, candidate.has_guard);
                 self.simplify_candidate(&mut candidate);
                 candidate
             })
@@ -147,18 +148,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match_pair: MatchPair<'pat, 'tcx>,
         candidate: &mut Candidate<'pat, 'tcx>,
     ) -> Result<(), MatchPair<'pat, 'tcx>> {
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
         match *match_pair.pattern.kind {
             PatKind::AscribeUserType {
                 ref subpattern,
                 ascription: thir::pattern::Ascription { variance, user_ty, user_ty_span },
             } => {
                 // Apply the type ascription to the value at `match_pair.place`, which is the
-                // value being matched, taking the variance field into account.
                 candidate.ascriptions.push(Ascription {
                     span: user_ty_span,
                     user_ty,
-                    source: match_pair.place,
+                    source: match_pair.place.clone().into_place(self.tcx, self.typeck_results),
                     variance,
                 });
 
@@ -177,7 +177,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     name,
                     mutability,
                     span: match_pair.pattern.span,
-                    source: match_pair.place,
+                    source: match_pair.place.clone().into_place(self.tcx, self.typeck_results),
                     var_id: var,
                     var_ty: ty,
                     binding_mode: mode,
@@ -251,21 +251,23 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             PatKind::Variant { adt_def, substs, variant_index, ref subpatterns } => {
                 let irrefutable = adt_def.variants.iter_enumerated().all(|(i, v)| {
                     i == variant_index || {
-                        self.hir.tcx().features().exhaustive_patterns
+                        self.tcx.features().exhaustive_patterns
                             && !v
                                 .uninhabited_from(
-                                    self.hir.tcx(),
+                                    self.tcx,
                                     substs,
                                     adt_def.adt_kind(),
-                                    self.hir.param_env,
+                                    self.param_env,
                                 )
                                 .is_empty()
                     }
                 }) && (adt_def.did.is_local()
                     || !adt_def.is_variant_list_non_exhaustive());
                 if irrefutable {
-                    let place = tcx.mk_place_downcast(match_pair.place, adt_def, variant_index);
-                    candidate.match_pairs.extend(self.field_match_pairs(place, subpatterns));
+                    let place_builder = match_pair.place.downcast(adt_def, variant_index);
+                    candidate
+                        .match_pairs
+                        .extend(self.field_match_pairs(place_builder, subpatterns));
                     Ok(())
                 } else {
                     Err(match_pair)
@@ -290,8 +292,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             PatKind::Deref { ref subpattern } => {
-                let place = tcx.mk_place_deref(match_pair.place);
-                candidate.match_pairs.push(MatchPair::new(place, subpattern));
+                let place_builder = match_pair.place.deref();
+                candidate.match_pairs.push(MatchPair::new(place_builder, subpattern));
                 Ok(())
             }
 
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 126fb957a6a..b082169cd63 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -5,6 +5,7 @@
 // identify what tests are needed, perform the tests, and then filter
 // the candidates based on the result.
 
+use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::{Candidate, MatchPair, Test, TestKind};
 use crate::build::Builder;
 use crate::thir::pattern::compare_const_vals;
@@ -13,9 +14,11 @@ use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::{LangItem, RangeEnd};
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::*;
+use rustc_middle::ty::subst::{GenericArg, Subst};
 use rustc_middle::ty::util::IntTypeExt;
-use rustc_middle::ty::{self, adjustment::PointerCast, Ty};
-use rustc_span::symbol::sym;
+use rustc_middle::ty::{self, adjustment::PointerCast, Ty, TyCtxt};
+use rustc_span::def_id::DefId;
+use rustc_span::symbol::{sym, Symbol};
 use rustc_target::abi::VariantIdx;
 
 use std::cmp::Ordering;
@@ -51,7 +54,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             PatKind::Constant { value } => Test {
                 span: match_pair.pattern.span,
-                kind: TestKind::Eq { value, ty: match_pair.pattern.ty.clone() },
+                kind: TestKind::Eq { value, ty: match_pair.pattern.ty },
             },
 
             PatKind::Range(range) => {
@@ -79,7 +82,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     pub(super) fn add_cases_to_switch<'pat>(
         &mut self,
-        test_place: &Place<'tcx>,
+        test_place: &PlaceBuilder<'tcx>,
         candidate: &Candidate<'pat, 'tcx>,
         switch_ty: Ty<'tcx>,
         options: &mut FxIndexMap<&'tcx ty::Const<'tcx>, u128>,
@@ -93,9 +96,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         match *match_pair.pattern.kind {
             PatKind::Constant { value } => {
-                options.entry(value).or_insert_with(|| {
-                    value.eval_bits(self.hir.tcx(), self.hir.param_env, switch_ty)
-                });
+                options
+                    .entry(value)
+                    .or_insert_with(|| value.eval_bits(self.tcx, self.param_env, switch_ty));
                 true
             }
             PatKind::Variant { .. } => {
@@ -121,7 +124,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     pub(super) fn add_variants_to_switch<'pat>(
         &mut self,
-        test_place: &Place<'tcx>,
+        test_place: &PlaceBuilder<'tcx>,
         candidate: &Candidate<'pat, 'tcx>,
         variants: &mut BitSet<VariantIdx>,
     ) -> bool {
@@ -149,15 +152,23 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     pub(super) fn perform_test(
         &mut self,
         block: BasicBlock,
-        place: Place<'tcx>,
+        place_builder: PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
         make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
     ) {
+        let place: Place<'tcx>;
+        if let Ok(test_place_builder) =
+            place_builder.try_upvars_resolved(self.tcx, self.typeck_results)
+        {
+            place = test_place_builder.into_place(self.tcx, self.typeck_results);
+        } else {
+            return;
+        }
         debug!(
             "perform_test({:?}, {:?}: {:?}, {:?})",
             block,
             place,
-            place.ty(&self.local_decls, self.hir.tcx()),
+            place.ty(&self.local_decls, self.tcx),
             test
         );
 
@@ -169,7 +180,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let num_enum_variants = adt_def.variants.len();
                 debug_assert_eq!(target_blocks.len(), num_enum_variants + 1);
                 let otherwise_block = *target_blocks.last().unwrap();
-                let tcx = self.hir.tcx();
+                let tcx = self.tcx;
                 let switch_targets = SwitchTargets::new(
                     adt_def.discriminants(tcx).filter_map(|(idx, discr)| {
                         if variants.contains(idx) {
@@ -217,7 +228,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             0 => (second_bb, first_bb),
                             v => span_bug!(test.span, "expected boolean value but got {:?}", v),
                         };
-                        TerminatorKind::if_(self.hir.tcx(), Operand::Copy(place), true_bb, false_bb)
+                        TerminatorKind::if_(self.tcx, Operand::Copy(place), true_bb, false_bb)
                     } else {
                         bug!("`TestKind::SwitchInt` on `bool` should have two targets")
                     }
@@ -292,7 +303,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             TestKind::Len { len, op } => {
                 let target_blocks = make_target_blocks(self);
 
-                let usize_ty = self.hir.usize_ty();
+                let usize_ty = self.tcx.types.usize;
                 let actual = self.temp(usize_ty, test.span);
 
                 // actual = len(place)
@@ -331,17 +342,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         left: Operand<'tcx>,
         right: Operand<'tcx>,
     ) {
-        let bool_ty = self.hir.bool_ty();
+        let bool_ty = self.tcx.types.bool;
         let result = self.temp(bool_ty, source_info.span);
 
         // result = op(left, right)
-        self.cfg.push_assign(block, source_info, result, Rvalue::BinaryOp(op, left, right));
+        self.cfg.push_assign(block, source_info, result, Rvalue::BinaryOp(op, box (left, right)));
 
         // branch based on result
         self.cfg.terminate(
             block,
             source_info,
-            TerminatorKind::if_(self.hir.tcx(), Operand::Move(result), success_block, fail_block),
+            TerminatorKind::if_(self.tcx, Operand::Move(result), success_block, fail_block),
         );
     }
 
@@ -377,7 +388,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // nothing to do, neither is an array
             (None, None) => {}
             (Some((region, elem_ty, _)), _) | (None, Some((region, elem_ty, _))) => {
-                let tcx = self.hir.tcx();
+                let tcx = self.tcx;
                 // make both a slice
                 ty = tcx.mk_imm_ref(region, tcx.mk_slice(elem_ty));
                 if opt_ref_ty.is_some() {
@@ -408,10 +419,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             _ => bug!("non_scalar_compare called on non-reference type: {}", ty),
         };
 
-        let eq_def_id = self.hir.tcx().require_lang_item(LangItem::PartialEq, None);
-        let method = self.hir.trait_method(eq_def_id, sym::eq, deref_ty, &[deref_ty.into()]);
+        let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, None);
+        let method = trait_method(self.tcx, eq_def_id, sym::eq, deref_ty, &[deref_ty.into()]);
 
-        let bool_ty = self.hir.bool_ty();
+        let bool_ty = self.tcx.types.bool;
         let eq_result = self.temp(bool_ty, source_info.span);
         let eq_block = self.cfg.start_new_block();
         self.cfg.terminate(
@@ -427,7 +438,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // Need to experiment.
                     user_ty: None,
 
-                    literal: method,
+                    literal: method.into(),
                 }),
                 args: vec![val, expect],
                 destination: Some((eq_result, eq_block)),
@@ -443,12 +454,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             self.cfg.terminate(
                 eq_block,
                 source_info,
-                TerminatorKind::if_(
-                    self.hir.tcx(),
-                    Operand::Move(eq_result),
-                    success_block,
-                    fail_block,
-                ),
+                TerminatorKind::if_(self.tcx, Operand::Move(eq_result), success_block, fail_block),
             );
         } else {
             bug!("`TestKind::Eq` should have two target blocks")
@@ -484,7 +490,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// tighter match code if we do something a bit different.
     pub(super) fn sort_candidate<'pat>(
         &mut self,
-        test_place: &Place<'tcx>,
+        test_place: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
         candidate: &mut Candidate<'pat, 'tcx>,
     ) -> Option<usize> {
@@ -632,11 +638,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     use rustc_hir::RangeEnd::*;
                     use std::cmp::Ordering::*;
 
-                    let tcx = self.hir.tcx();
+                    let tcx = self.tcx;
 
                     let test_ty = test.lo.ty;
-                    let lo = compare_const_vals(tcx, test.lo, pat.hi, self.hir.param_env, test_ty)?;
-                    let hi = compare_const_vals(tcx, test.hi, pat.lo, self.hir.param_env, test_ty)?;
+                    let lo = compare_const_vals(tcx, test.lo, pat.hi, self.param_env, test_ty)?;
+                    let hi = compare_const_vals(tcx, test.hi, pat.lo, self.param_env, test_ty)?;
 
                     match (test.end, pat.end, lo, hi) {
                         // pat < test
@@ -731,7 +737,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         candidate: &mut Candidate<'pat, 'tcx>,
     ) {
         let match_pair = candidate.match_pairs.remove(match_pair_index);
-        let tcx = self.hir.tcx();
 
         // So, if we have a match-pattern like `x @ Enum::Variant(P1, P2)`,
         // we want to create a set of derived match-patterns like
@@ -740,10 +745,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             Some(adt_def.variants[variant_index].ident.name),
             variant_index,
         );
-        let downcast_place = tcx.mk_place_elem(match_pair.place, elem); // `(x as Variant)`
+        let downcast_place = match_pair.place.project(elem); // `(x as Variant)`
         let consequent_match_pairs = subpatterns.iter().map(|subpattern| {
             // e.g., `(x as Variant).0`
-            let place = tcx.mk_place_field(downcast_place, subpattern.field, subpattern.pattern.ty);
+            let place = downcast_place.clone().field(subpattern.field, subpattern.pattern.ty);
             // e.g., `(x as Variant).0 @ P1`
             MatchPair::new(place, &subpattern.pattern)
         });
@@ -762,10 +767,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ) -> Option<bool> {
         use std::cmp::Ordering::*;
 
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
 
-        let a = compare_const_vals(tcx, range.lo, value, self.hir.param_env, range.lo.ty)?;
-        let b = compare_const_vals(tcx, value, range.hi, self.hir.param_env, range.lo.ty)?;
+        let a = compare_const_vals(tcx, range.lo, value, self.param_env, range.lo.ty)?;
+        let b = compare_const_vals(tcx, value, range.hi, self.param_env, range.lo.ty)?;
 
         match (b, range.end) {
             (Less, _) | (Equal, RangeEnd::Included) if a != Greater => Some(true),
@@ -815,3 +820,25 @@ impl Test<'_> {
 fn is_switch_ty(ty: Ty<'_>) -> bool {
     ty.is_integral() || ty.is_char() || ty.is_bool()
 }
+
+fn trait_method<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    trait_def_id: DefId,
+    method_name: Symbol,
+    self_ty: Ty<'tcx>,
+    params: &[GenericArg<'tcx>],
+) -> &'tcx ty::Const<'tcx> {
+    let substs = tcx.mk_substs_trait(self_ty, params);
+
+    // The unhygienic comparison here is acceptable because this is only
+    // used on known traits.
+    let item = tcx
+        .associated_items(trait_def_id)
+        .filter_by_name_unhygienic(method_name)
+        .find(|item| item.kind == ty::AssocKind::Fn)
+        .expect("trait method not found");
+
+    let method_ty = tcx.type_of(item.def_id);
+    let method_ty = method_ty.subst(tcx, substs);
+    ty::Const::zero_sized(tcx, method_ty)
+}
diff --git a/compiler/rustc_mir_build/src/build/matches/util.rs b/compiler/rustc_mir_build/src/build/matches/util.rs
index db1f678a5c6..d49a00a5660 100644
--- a/compiler/rustc_mir_build/src/build/matches/util.rs
+++ b/compiler/rustc_mir_build/src/build/matches/util.rs
@@ -1,3 +1,4 @@
+use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::MatchPair;
 use crate::build::Builder;
 use crate::thir::*;
@@ -9,14 +10,13 @@ use std::convert::TryInto;
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     crate fn field_match_pairs<'pat>(
         &mut self,
-        place: Place<'tcx>,
+        place: PlaceBuilder<'tcx>,
         subpatterns: &'pat [FieldPat<'tcx>],
     ) -> Vec<MatchPair<'pat, 'tcx>> {
         subpatterns
             .iter()
             .map(|fieldpat| {
-                let place =
-                    self.hir.tcx().mk_place_field(place, fieldpat.field, fieldpat.pattern.ty);
+                let place = place.clone().field(fieldpat.field, fieldpat.pattern.ty);
                 MatchPair::new(place, &fieldpat.pattern)
             })
             .collect()
@@ -25,34 +25,37 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     crate fn prefix_slice_suffix<'pat>(
         &mut self,
         match_pairs: &mut SmallVec<[MatchPair<'pat, 'tcx>; 1]>,
-        place: &Place<'tcx>,
+        place: &PlaceBuilder<'tcx>,
         prefix: &'pat [Pat<'tcx>],
         opt_slice: Option<&'pat Pat<'tcx>>,
         suffix: &'pat [Pat<'tcx>],
     ) {
-        let tcx = self.hir.tcx();
-        let (min_length, exact_size) = match place.ty(&self.local_decls, tcx).ty.kind() {
-            ty::Array(_, length) => (length.eval_usize(tcx, self.hir.param_env), true),
+        let tcx = self.tcx;
+        let (min_length, exact_size) = match place
+            .clone()
+            .into_place(tcx, self.typeck_results)
+            .ty(&self.local_decls, tcx)
+            .ty
+            .kind()
+        {
+            ty::Array(_, length) => (length.eval_usize(tcx, self.param_env), true),
             _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
         };
 
         match_pairs.extend(prefix.iter().enumerate().map(|(idx, subpattern)| {
             let elem =
                 ProjectionElem::ConstantIndex { offset: idx as u64, min_length, from_end: false };
-            let place = tcx.mk_place_elem(*place, elem);
+            let place = place.clone().project(elem);
             MatchPair::new(place, subpattern)
         }));
 
         if let Some(subslice_pat) = opt_slice {
             let suffix_len = suffix.len() as u64;
-            let subslice = tcx.mk_place_elem(
-                *place,
-                ProjectionElem::Subslice {
-                    from: prefix.len() as u64,
-                    to: if exact_size { min_length - suffix_len } else { suffix_len },
-                    from_end: !exact_size,
-                },
-            );
+            let subslice = place.clone().project(ProjectionElem::Subslice {
+                from: prefix.len() as u64,
+                to: if exact_size { min_length - suffix_len } else { suffix_len },
+                from_end: !exact_size,
+            });
             match_pairs.push(MatchPair::new(subslice, subslice_pat));
         }
 
@@ -63,7 +66,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 min_length,
                 from_end: !exact_size,
             };
-            let place = tcx.mk_place_elem(*place, elem);
+            let place = place.clone().project(elem);
             MatchPair::new(place, subpattern)
         }));
     }
@@ -92,7 +95,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 }
 
 impl<'pat, 'tcx> MatchPair<'pat, 'tcx> {
-    crate fn new(place: Place<'tcx>, pattern: &'pat Pat<'tcx>) -> MatchPair<'pat, 'tcx> {
+    crate fn new(place: PlaceBuilder<'tcx>, pattern: &'pat Pat<'tcx>) -> MatchPair<'pat, 'tcx> {
         MatchPair { place, pattern }
     }
 }
diff --git a/compiler/rustc_mir_build/src/build/misc.rs b/compiler/rustc_mir_build/src/build/misc.rs
index 29651d9bc66..a1126d1c3d5 100644
--- a/compiler/rustc_mir_build/src/build/misc.rs
+++ b/compiler/rustc_mir_build/src/build/misc.rs
@@ -3,10 +3,10 @@
 
 use crate::build::Builder;
 
-use rustc_middle::ty::{self, Ty};
-
 use rustc_middle::mir::*;
+use rustc_middle::ty::{self, Ty};
 use rustc_span::{Span, DUMMY_SP};
+use rustc_trait_selection::infer::InferCtxtExt;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Adds a new temporary value of type `ty` storing the result of
@@ -30,6 +30,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         span: Span,
         literal: &'tcx ty::Const<'tcx>,
     ) -> Operand<'tcx> {
+        let literal = literal.into();
         let constant = box Constant { span, user_ty: None, literal };
         Operand::Constant(constant)
     }
@@ -37,7 +38,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     // Returns a zero literal operand for the appropriate type, works for
     // bool, char and integers.
     crate fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
-        let literal = ty::Const::from_bits(self.hir.tcx(), 0, ty::ParamEnv::empty().and(ty));
+        let literal = ty::Const::from_bits(self.tcx, 0, ty::ParamEnv::empty().and(ty));
 
         self.literal_operand(span, literal)
     }
@@ -48,7 +49,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         source_info: SourceInfo,
         value: u64,
     ) -> Place<'tcx> {
-        let usize_ty = self.hir.usize_ty();
+        let usize_ty = self.tcx.types.usize;
         let temp = self.temp(usize_ty, source_info.span);
         self.cfg.push_assign_constant(
             block,
@@ -57,16 +58,16 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             Constant {
                 span: source_info.span,
                 user_ty: None,
-                literal: self.hir.usize_literal(value),
+                literal: ty::Const::from_usize(self.tcx, value).into(),
             },
         );
         temp
     }
 
     crate fn consume_by_copy_or_move(&self, place: Place<'tcx>) -> Operand<'tcx> {
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
         let ty = place.ty(&self.local_decls, tcx).ty;
-        if !self.hir.type_is_copy_modulo_regions(ty, DUMMY_SP) {
+        if !self.infcx.type_is_copy_modulo_regions(self.param_env, ty, DUMMY_SP) {
             Operand::Move(place)
         } else {
             Operand::Copy(place)
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index b928458df8e..f944e5f8f04 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -1,7 +1,7 @@
 use crate::build;
+use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::scope::DropKind;
-use crate::thir::cx::Cx;
-use crate::thir::{BindingMode, LintLevel, PatKind};
+use crate::thir::{build_thir, Arena, BindingMode, Expr, LintLevel, Pat, PatKind};
 use rustc_attr::{self as attr, UnwindAttr};
 use rustc_errors::ErrorReported;
 use rustc_hir as hir;
@@ -9,13 +9,13 @@ use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{GeneratorKind, HirIdMap, Node};
 use rustc_index::vec::{Idx, IndexVec};
-use rustc_infer::infer::TyCtxtInferExt;
+use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
 use rustc_middle::ty::subst::Subst;
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable};
-use rustc_span::symbol::kw;
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeckResults};
+use rustc_span::symbol::{kw, sym};
 use rustc_span::Span;
 use rustc_target::spec::abi::Abi;
 use rustc_target::spec::PanicStrategy;
@@ -42,6 +42,8 @@ crate fn mir_built<'tcx>(
 /// Construct the MIR for a given `DefId`.
 fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> {
     let id = tcx.hir().local_def_id_to_hir_id(def.did);
+    let body_owner_kind = tcx.hir().body_owner_kind(id);
+    let typeck_results = tcx.typeck_opt_const_arg(def);
 
     // Figure out what primary body this item has.
     let (body_id, return_ty_span, span_with_body) = match tcx.hir().get(id) {
@@ -86,15 +88,15 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
     // If we don't have a specialized span for the body, just use the
     // normal def span.
     let span_with_body = span_with_body.unwrap_or_else(|| tcx.hir().span(id));
+    let arena = Arena::default();
 
     tcx.infer_ctxt().enter(|infcx| {
-        let cx = Cx::new(&infcx, def, id);
-        let body = if let Some(ErrorReported) = cx.typeck_results().tainted_by_errors {
-            build::construct_error(cx, body_id)
-        } else if cx.body_owner_kind.is_fn_or_closure() {
+        let body = if let Some(ErrorReported) = typeck_results.tainted_by_errors {
+            build::construct_error(&infcx, def, id, body_id, body_owner_kind)
+        } else if body_owner_kind.is_fn_or_closure() {
             // fetch the fully liberated fn signature (that is, all bound
             // types/lifetimes replaced)
-            let fn_sig = cx.typeck_results().liberated_fn_sigs()[id];
+            let fn_sig = typeck_results.liberated_fn_sigs()[id];
             let fn_def_id = tcx.hir().local_def_id(id);
 
             let safety = match fn_sig.unsafety {
@@ -103,6 +105,7 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
             };
 
             let body = tcx.hir().body(body_id);
+            let thir = build_thir(tcx, def, &arena, &body.value);
             let ty = tcx.type_of(fn_def_id);
             let mut abi = fn_sig.abi;
             let implicit_argument = match ty.kind() {
@@ -178,7 +181,8 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
             };
 
             let mut mir = build::construct_fn(
-                cx,
+                &infcx,
+                def,
                 id,
                 arguments,
                 safety,
@@ -186,6 +190,7 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
                 return_ty,
                 return_ty_span,
                 body,
+                thir,
                 span_with_body,
             );
             if yield_ty.is_some() {
@@ -205,9 +210,12 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
             // place to be the type of the constant because NLL typeck will
             // equate them.
 
-            let return_ty = cx.typeck_results().node_type(id);
+            let return_ty = typeck_results.node_type(id);
 
-            build::construct_const(cx, body_id, return_ty, return_ty_span)
+            let ast_expr = &tcx.hir().body(body_id).value;
+            let thir = build_thir(tcx, def, &arena, ast_expr);
+
+            build::construct_const(&infcx, thir, def, id, return_ty, return_ty_span)
         };
 
         lints::check(tcx, &body);
@@ -244,8 +252,13 @@ fn liberated_closure_env_ty(
         _ => bug!("closure expr does not have closure type: {:?}", closure_ty),
     };
 
-    let closure_env_ty = tcx.closure_env_ty(closure_def_id, closure_substs).unwrap();
-    tcx.erase_late_bound_regions(closure_env_ty)
+    let bound_vars =
+        tcx.mk_bound_variable_kinds(std::iter::once(ty::BoundVariableKind::Region(ty::BrEnv)));
+    let br =
+        ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BrEnv };
+    let env_region = ty::ReLateBound(ty::INNERMOST, br);
+    let closure_env_ty = tcx.closure_env_ty(closure_def_id, closure_substs, env_region).unwrap();
+    tcx.erase_late_bound_regions(ty::Binder::bind_with_vars(closure_env_ty, bound_vars))
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -304,10 +317,17 @@ impl BlockFrame {
 struct BlockContext(Vec<BlockFrame>);
 
 struct Builder<'a, 'tcx> {
-    hir: Cx<'a, 'tcx>,
+    tcx: TyCtxt<'tcx>,
+    infcx: &'a InferCtxt<'a, 'tcx>,
+    typeck_results: &'tcx TypeckResults<'tcx>,
+    region_scope_tree: &'tcx region::ScopeTree,
+    param_env: ty::ParamEnv<'tcx>,
+
     cfg: CFG<'tcx>,
 
     def_id: DefId,
+    hir_id: hir::HirId,
+    check_overflow: bool,
     fn_span: Span,
     arg_count: usize,
     generator_kind: Option<GeneratorKind>,
@@ -548,7 +568,7 @@ macro_rules! unpack {
     }};
 }
 
-fn should_abort_on_panic(tcx: TyCtxt<'_>, fn_def_id: LocalDefId, _abi: Abi) -> bool {
+fn should_abort_on_panic(tcx: TyCtxt<'_>, fn_def_id: LocalDefId, abi: Abi) -> bool {
     // Validate `#[unwind]` syntax regardless of platform-specific panic strategy.
     let attrs = &tcx.get_attrs(fn_def_id.to_def_id());
     let unwind_attr = attr::find_unwind_attr(&tcx.sess, attrs);
@@ -558,12 +578,47 @@ fn should_abort_on_panic(tcx: TyCtxt<'_>, fn_def_id: LocalDefId, _abi: Abi) -> b
         return false;
     }
 
-    // This is a special case: some functions have a C abi but are meant to
-    // unwind anyway. Don't stop them.
     match unwind_attr {
-        None => false, // FIXME(#58794); should be `!(abi == Abi::Rust || abi == Abi::RustCall)`
+        // If an `#[unwind]` attribute was found, we should adhere to it.
         Some(UnwindAttr::Allowed) => false,
         Some(UnwindAttr::Aborts) => true,
+        // If no attribute was found and the panic strategy is `unwind`, then we should examine
+        // the function's ABI string to determine whether it should abort upon panic.
+        None if tcx.features().c_unwind => {
+            use Abi::*;
+            match abi {
+                // In the case of ABI's that have an `-unwind` equivalent, check whether the ABI
+                // permits unwinding. If so, we should not abort. Otherwise, we should.
+                C { unwind } | Stdcall { unwind } | System { unwind } | Thiscall { unwind } => {
+                    !unwind
+                }
+                // Rust and `rust-call` functions are allowed to unwind, and should not abort.
+                Rust | RustCall => false,
+                // Other ABI's should abort.
+                Cdecl
+                | Fastcall
+                | Vectorcall
+                | Aapcs
+                | Win64
+                | SysV64
+                | PtxKernel
+                | Msp430Interrupt
+                | X86Interrupt
+                | AmdGpuKernel
+                | EfiApi
+                | AvrInterrupt
+                | AvrNonBlockingInterrupt
+                | CCmseNonSecureCall
+                | Wasm
+                | RustIntrinsic
+                | PlatformIntrinsic
+                | Unadjusted => true,
+            }
+        }
+        // If the `c_unwind` feature gate is not active, follow the behavior that was in place
+        // prior to #76570. This is a special case: some functions have a C ABI but are meant to
+        // unwind anyway. Don't stop them.
+        None => false, // FIXME(#58794); should be `!(abi == Abi::Rust || abi == Abi::RustCall)`
     }
 }
 
@@ -577,8 +632,9 @@ struct ArgInfo<'tcx>(
     Option<ImplicitSelfKind>,
 );
 
-fn construct_fn<'a, 'tcx, A>(
-    hir: Cx<'a, 'tcx>,
+fn construct_fn<'tcx, A>(
+    infcx: &InferCtxt<'_, 'tcx>,
+    fn_def: ty::WithOptConstParam<LocalDefId>,
     fn_id: hir::HirId,
     arguments: A,
     safety: Safety,
@@ -586,6 +642,7 @@ fn construct_fn<'a, 'tcx, A>(
     return_ty: Ty<'tcx>,
     return_ty_span: Span,
     body: &'tcx hir::Body<'tcx>,
+    expr: &Expr<'_, 'tcx>,
     span_with_body: Span,
 ) -> Body<'tcx>
 where
@@ -593,15 +650,13 @@ where
 {
     let arguments: Vec<_> = arguments.collect();
 
-    let tcx = hir.tcx();
-    let tcx_hir = tcx.hir();
-    let span = tcx_hir.span(fn_id);
-
-    let fn_def_id = tcx_hir.local_def_id(fn_id);
+    let tcx = infcx.tcx;
+    let span = tcx.hir().span(fn_id);
 
     let mut builder = Builder::new(
-        hir,
-        fn_def_id.to_def_id(),
+        infcx,
+        fn_def,
+        fn_id,
         span_with_body,
         arguments.len(),
         safety,
@@ -625,16 +680,16 @@ where
                 Some(builder.in_scope(arg_scope_s, LintLevel::Inherited, |builder| {
                     builder.args_and_body(
                         START_BLOCK,
-                        fn_def_id.to_def_id(),
+                        fn_def.did.to_def_id(),
                         &arguments,
                         arg_scope,
-                        &body.value,
+                        expr,
                     )
                 }))
             }));
         let source_info = builder.source_info(fn_end);
         builder.cfg.terminate(return_block, source_info, TerminatorKind::Return);
-        let should_abort = should_abort_on_panic(tcx, fn_def_id, abi);
+        let should_abort = should_abort_on_panic(tcx, fn_def.did, abi);
         builder.build_drop_trees(should_abort);
         return_block.unit()
     }));
@@ -645,7 +700,7 @@ where
     } else {
         None
     };
-    debug!("fn_id {:?} has attrs {:?}", fn_def_id, tcx.get_attrs(fn_def_id.to_def_id()));
+    debug!("fn_id {:?} has attrs {:?}", fn_def, tcx.get_attrs(fn_def.did.to_def_id()));
 
     let mut body = builder.finish();
     body.spread_arg = spread_arg;
@@ -653,22 +708,20 @@ where
 }
 
 fn construct_const<'a, 'tcx>(
-    hir: Cx<'a, 'tcx>,
-    body_id: hir::BodyId,
+    infcx: &'a InferCtxt<'a, 'tcx>,
+    expr: &Expr<'_, 'tcx>,
+    def: ty::WithOptConstParam<LocalDefId>,
+    hir_id: hir::HirId,
     const_ty: Ty<'tcx>,
     const_ty_span: Span,
 ) -> Body<'tcx> {
-    let tcx = hir.tcx();
-    let owner_id = tcx.hir().body_owner(body_id);
-    let def_id = tcx.hir().local_def_id(owner_id);
-    let span = tcx.hir().span(owner_id);
+    let tcx = infcx.tcx;
+    let span = tcx.hir().span(hir_id);
     let mut builder =
-        Builder::new(hir, def_id.to_def_id(), span, 0, Safety::Safe, const_ty, const_ty_span, None);
+        Builder::new(infcx, def, hir_id, span, 0, Safety::Safe, const_ty, const_ty_span, None);
 
     let mut block = START_BLOCK;
-    let ast_expr = &tcx.hir().body(body_id).value;
-    let expr = builder.hir.mirror(ast_expr);
-    unpack!(block = builder.into_expr(Place::return_place(), block, expr));
+    unpack!(block = builder.expr_into_dest(Place::return_place(), block, &expr));
 
     let source_info = builder.source_info(span);
     builder.cfg.terminate(block, source_info, TerminatorKind::Return);
@@ -682,15 +735,19 @@ fn construct_const<'a, 'tcx>(
 ///
 /// This is required because we may still want to run MIR passes on an item
 /// with type errors, but normal MIR construction can't handle that in general.
-fn construct_error<'a, 'tcx>(hir: Cx<'a, 'tcx>, body_id: hir::BodyId) -> Body<'tcx> {
-    let tcx = hir.tcx();
-    let owner_id = tcx.hir().body_owner(body_id);
-    let def_id = tcx.hir().local_def_id(owner_id);
-    let span = tcx.hir().span(owner_id);
+fn construct_error<'a, 'tcx>(
+    infcx: &'a InferCtxt<'a, 'tcx>,
+    def: ty::WithOptConstParam<LocalDefId>,
+    hir_id: hir::HirId,
+    body_id: hir::BodyId,
+    body_owner_kind: hir::BodyOwnerKind,
+) -> Body<'tcx> {
+    let tcx = infcx.tcx;
+    let span = tcx.hir().span(hir_id);
     let ty = tcx.ty_error();
     let generator_kind = tcx.hir().body(body_id).generator_kind;
-    let num_params = match hir.body_owner_kind {
-        hir::BodyOwnerKind::Fn => tcx.hir().fn_decl_by_hir_id(owner_id).unwrap().inputs.len(),
+    let num_params = match body_owner_kind {
+        hir::BodyOwnerKind::Fn => tcx.hir().fn_decl_by_hir_id(hir_id).unwrap().inputs.len(),
         hir::BodyOwnerKind::Closure => {
             if generator_kind.is_some() {
                 // Generators have an implicit `self` parameter *and* a possibly
@@ -698,22 +755,14 @@ fn construct_error<'a, 'tcx>(hir: Cx<'a, 'tcx>, body_id: hir::BodyId) -> Body<'t
                 2
             } else {
                 // The implicit self parameter adds another local in MIR.
-                1 + tcx.hir().fn_decl_by_hir_id(owner_id).unwrap().inputs.len()
+                1 + tcx.hir().fn_decl_by_hir_id(hir_id).unwrap().inputs.len()
             }
         }
         hir::BodyOwnerKind::Const => 0,
         hir::BodyOwnerKind::Static(_) => 0,
     };
-    let mut builder = Builder::new(
-        hir,
-        def_id.to_def_id(),
-        span,
-        num_params,
-        Safety::Safe,
-        ty,
-        span,
-        generator_kind,
-    );
+    let mut builder =
+        Builder::new(infcx, def, hir_id, span, num_params, Safety::Safe, ty, span, generator_kind);
     let source_info = builder.source_info(span);
     // Some MIR passes will expect the number of parameters to match the
     // function declaration.
@@ -728,8 +777,9 @@ fn construct_error<'a, 'tcx>(hir: Cx<'a, 'tcx>, body_id: hir::BodyId) -> Body<'t
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn new(
-        hir: Cx<'a, 'tcx>,
-        def_id: DefId,
+        infcx: &'a InferCtxt<'a, 'tcx>,
+        def: ty::WithOptConstParam<LocalDefId>,
+        hir_id: hir::HirId,
         span: Span,
         arg_count: usize,
         safety: Safety,
@@ -737,10 +787,30 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         return_span: Span,
         generator_kind: Option<GeneratorKind>,
     ) -> Builder<'a, 'tcx> {
-        let lint_level = LintLevel::Explicit(hir.root_lint_level);
+        let tcx = infcx.tcx;
+        let attrs = tcx.hir().attrs(hir_id);
+        // Some functions always have overflow checks enabled,
+        // however, they may not get codegen'd, depending on
+        // the settings for the crate they are codegened in.
+        let mut check_overflow = tcx.sess.contains_name(attrs, sym::rustc_inherit_overflow_checks);
+        // Respect -C overflow-checks.
+        check_overflow |= tcx.sess.overflow_checks();
+        // Constants always need overflow checks.
+        check_overflow |= matches!(
+            tcx.hir().body_owner_kind(hir_id),
+            hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_)
+        );
+
+        let lint_level = LintLevel::Explicit(hir_id);
         let mut builder = Builder {
-            hir,
-            def_id,
+            tcx,
+            infcx,
+            typeck_results: tcx.typeck_opt_const_arg(def),
+            region_scope_tree: tcx.region_scope_tree(def.did),
+            param_env: tcx.param_env(def.did),
+            def_id: def.did.to_def_id(),
+            hir_id,
+            check_overflow,
             cfg: CFG { basic_blocks: IndexVec::new() },
             fn_span: span,
             arg_count,
@@ -796,7 +866,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         fn_def_id: DefId,
         arguments: &[ArgInfo<'tcx>],
         argument_scope: region::Scope,
-        ast_body: &'tcx hir::Expr<'tcx>,
+        expr: &Expr<'_, 'tcx>,
     ) -> BlockAnd<()> {
         // Allocate locals for the function arguments
         for &ArgInfo(ty, _, arg_opt, _) in arguments.iter() {
@@ -816,12 +886,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
         }
 
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
         let tcx_hir = tcx.hir();
-        let hir_typeck_results = self.hir.typeck_results();
+        let hir_typeck_results = self.typeck_results;
 
         // In analyze_closure() in upvar.rs we gathered a list of upvars used by a
-        // indexed closure and we stored in a map called closure_captures in TypeckResults
+        // indexed closure and we stored in a map called closure_min_captures in TypeckResults
         // with the closure's DefId. Here, we run through that vec of UpvarIds for
         // the given closure and use the necessary information to create upvar
         // debuginfo and to fill `self.upvar_mutbls`.
@@ -894,14 +964,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             // Make sure we drop (parts of) the argument even when not matched on.
             self.schedule_drop(
-                arg_opt.as_ref().map_or(ast_body.span, |arg| arg.pat.span),
+                arg_opt.as_ref().map_or(expr.span, |arg| arg.pat.span),
                 argument_scope,
                 local,
                 DropKind::Value,
             );
 
             if let Some(arg) = arg_opt {
-                let pattern = self.hir.pattern_from_hir(&arg.pat);
+                let pat = match tcx.hir().get(arg.pat.hir_id) {
+                    Node::Pat(pat) | Node::Binding(pat) => pat,
+                    node => bug!("pattern became {:?}", node),
+                };
+                let pattern = Pat::from_hir(tcx, self.param_env, self.typeck_results, pat);
                 let original_source_scope = self.source_scope;
                 let span = pattern.span;
                 self.set_correct_source_scope_for_arg(arg.hir_id, original_source_scope, span);
@@ -936,12 +1010,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     _ => {
                         scope = self.declare_bindings(
                             scope,
-                            ast_body.span,
+                            expr.span,
                             &pattern,
                             matches::ArmHasGuard(false),
                             Some((Some(&place), span)),
                         );
-                        unpack!(block = self.place_into_pattern(block, pattern, place, false));
+                        let place_builder = PlaceBuilder::from(local);
+                        unpack!(
+                            block = self.place_into_pattern(block, pattern, place_builder, false)
+                        );
                     }
                 }
                 self.source_scope = original_source_scope;
@@ -953,8 +1030,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             self.source_scope = source_scope;
         }
 
-        let body = self.hir.mirror(ast_body);
-        self.into(Place::return_place(), block, body)
+        self.expr_into_dest(Place::return_place(), block, &expr)
     }
 
     fn set_correct_source_scope_for_arg(
@@ -963,15 +1039,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         original_source_scope: SourceScope,
         pattern_span: Span,
     ) {
-        let tcx = self.hir.tcx();
-        let current_root = tcx.maybe_lint_level_root_bounded(arg_hir_id, self.hir.root_lint_level);
+        let tcx = self.tcx;
+        let current_root = tcx.maybe_lint_level_root_bounded(arg_hir_id, self.hir_id);
         let parent_root = tcx.maybe_lint_level_root_bounded(
             self.source_scopes[original_source_scope]
                 .local_data
                 .as_ref()
                 .assert_crate_local()
                 .lint_root,
-            self.hir.root_lint_level,
+            self.hir_id,
         );
         if current_root != parent_root {
             self.source_scope =
@@ -983,7 +1059,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match self.unit_temp {
             Some(tmp) => tmp,
             None => {
-                let ty = self.hir.unit_ty();
+                let ty = self.tcx.mk_unit();
                 let fn_span = self.fn_span;
                 let tmp = self.temp(ty, fn_span);
                 self.unit_temp = Some(tmp);
@@ -1001,7 +1077,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 mod block;
 mod cfg;
 mod expr;
-mod into;
 mod matches;
 mod misc;
 mod scope;
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index 5e9d780d179..41fc925c049 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -82,7 +82,7 @@ that contains only loops and breakable blocks. It tracks where a `break`,
 */
 
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
-use crate::thir::{Expr, ExprRef, LintLevel};
+use crate::thir::{Expr, LintLevel};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::vec::IndexVec;
 use rustc_middle::middle::region;
@@ -516,7 +516,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     {
         debug!("in_scope(region_scope={:?})", region_scope);
         let source_scope = self.source_scope;
-        let tcx = self.hir.tcx();
+        let tcx = self.tcx;
         if let LintLevel::Explicit(current_hir_id) = lint_level {
             // Use `maybe_lint_level_root_bounded` with `root_lint_level` as a bound
             // to avoid adding Hir dependences on our parents.
@@ -524,10 +524,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             let parent_root = tcx.maybe_lint_level_root_bounded(
                 self.source_scopes[source_scope].local_data.as_ref().assert_crate_local().lint_root,
-                self.hir.root_lint_level,
+                self.hir_id,
             );
-            let current_root =
-                tcx.maybe_lint_level_root_bounded(current_hir_id, self.hir.root_lint_level);
+            let current_root = tcx.maybe_lint_level_root_bounded(current_hir_id, self.hir_id);
 
             if parent_root != current_root {
                 self.source_scope = self.new_source_scope(
@@ -575,7 +574,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     crate fn break_scope(
         &mut self,
         mut block: BasicBlock,
-        value: Option<ExprRef<'tcx>>,
+        value: Option<&Expr<'_, 'tcx>>,
         target: BreakableTarget,
         source_info: SourceInfo,
     ) -> BlockAnd<()> {
@@ -612,13 +611,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             if let Some(value) = value {
                 debug!("stmt_expr Break val block_context.push(SubExpr)");
                 self.block_context.push(BlockFrame::SubExpr);
-                unpack!(block = self.into(destination, block, value));
+                unpack!(block = self.expr_into_dest(destination, block, value));
                 self.block_context.pop();
             } else {
-                self.cfg.push_assign_unit(block, source_info, destination, self.hir.tcx())
+                self.cfg.push_assign_unit(block, source_info, destination, self.tcx)
             }
         } else {
             assert!(value.is_none(), "`return` and `break` should have a destination");
+            if self.tcx.sess.instrument_coverage() {
+                // Unlike `break` and `return`, which push an `Assign` statement to MIR, from which
+                // a Coverage code region can be generated, `continue` needs no `Assign`; but
+                // without one, the `InstrumentCoverage` MIR pass cannot generate a code region for
+                // `continue`. Coverage will be missing unless we add a dummy `Assign` to MIR.
+                self.add_dummy_assignment(&span, block, source_info);
+            }
         }
 
         let region_scope = self.scopes.breakable_scopes[break_index].region_scope;
@@ -644,6 +650,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         self.cfg.start_new_block().unit()
     }
 
+    // Add a dummy `Assign` statement to the CFG, with the span for the source code's `continue`
+    // statement.
+    fn add_dummy_assignment(&mut self, span: &Span, block: BasicBlock, source_info: SourceInfo) {
+        let local_decl = LocalDecl::new(self.tcx.mk_unit(), *span).internal();
+        let temp_place = Place::from(self.local_decls.push(local_decl));
+        self.cfg.push_assign_unit(block, source_info, temp_place, self.tcx);
+    }
+
     crate fn exit_top_scope(
         &mut self,
         mut block: BasicBlock,
@@ -763,7 +777,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ) {
         let needs_drop = match drop_kind {
             DropKind::Value => {
-                if !self.hir.needs_drop(self.local_decls[local].ty) {
+                if !self.local_decls[local].ty.needs_drop(self.tcx, self.param_env) {
                     return;
                 }
                 true
@@ -834,10 +848,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             if scope.region_scope == region_scope {
-                let region_scope_span =
-                    region_scope.span(self.hir.tcx(), &self.hir.region_scope_tree);
+                let region_scope_span = region_scope.span(self.tcx, &self.region_scope_tree);
                 // Attribute scope exit drops to scope's closing brace.
-                let scope_end = self.hir.tcx().sess.source_map().end_point(region_scope_span);
+                let scope_end = self.tcx.sess.source_map().end_point(region_scope_span);
 
                 scope.drops.push(DropData {
                     source_info: SourceInfo { span: scope_end, scope: scope.source_scope },
@@ -920,13 +933,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     crate fn test_bool(
         &mut self,
         mut block: BasicBlock,
-        condition: Expr<'tcx>,
+        condition: &Expr<'_, 'tcx>,
         source_info: SourceInfo,
     ) -> (BasicBlock, BasicBlock) {
         let cond = unpack!(block = self.as_local_operand(block, condition));
         let true_block = self.cfg.start_new_block();
         let false_block = self.cfg.start_new_block();
-        let term = TerminatorKind::if_(self.hir.tcx(), cond.clone(), true_block, false_block);
+        let term = TerminatorKind::if_(self.tcx, cond.clone(), true_block, false_block);
         self.cfg.terminate(block, source_info, term);
 
         match cond {
diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs
index 0866892265b..23bc1da09b5 100644
--- a/compiler/rustc_mir_build/src/lib.rs
+++ b/compiler/rustc_mir_build/src/lib.rs
@@ -9,8 +9,9 @@
 #![feature(control_flow_enum)]
 #![feature(crate_visibility_modifier)]
 #![feature(bool_to_option)]
+#![feature(iter_zip)]
 #![feature(once_cell)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 #[macro_use]
@@ -20,7 +21,7 @@ extern crate rustc_middle;
 
 mod build;
 mod lints;
-mod thir;
+pub mod thir;
 
 use rustc_middle::ty::query::Providers;
 
diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs
index 576b537c017..ef8bd20d510 100644
--- a/compiler/rustc_mir_build/src/lints.rs
+++ b/compiler/rustc_mir_build/src/lints.rs
@@ -15,7 +15,7 @@ crate fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
     let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
 
     if let Some(fn_like_node) = FnLikeNode::from_node(tcx.hir().get(hir_id)) {
-        if let FnKind::Closure(_) = fn_like_node.kind() {
+        if let FnKind::Closure = fn_like_node.kind() {
             // closures can't recur, so they don't matter.
             return;
         }
diff --git a/compiler/rustc_mir_build/src/thir/arena.rs b/compiler/rustc_mir_build/src/thir/arena.rs
new file mode 100644
index 00000000000..aacc7b12a42
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/arena.rs
@@ -0,0 +1,98 @@
+use crate::thir::*;
+
+macro_rules! declare_arena {
+    ([], [$($a:tt $name:ident: $ty:ty,)*]) => {
+        #[derive(Default)]
+        pub struct Arena<'thir, 'tcx> {
+            pub dropless: rustc_arena::DroplessArena,
+            drop: rustc_arena::DropArena,
+            $($name: rustc_arena::arena_for_type!($a[$ty]),)*
+        }
+
+        pub trait ArenaAllocatable<'thir, 'tcx, T = Self>: Sized {
+            fn allocate_on(self, arena: &'thir Arena<'thir, 'tcx>) -> &'thir mut Self;
+            fn allocate_from_iter(
+                arena: &'thir Arena<'thir, 'tcx>,
+                iter: impl ::std::iter::IntoIterator<Item = Self>,
+            ) -> &'thir mut [Self];
+        }
+
+        impl<'thir, 'tcx, T: Copy> ArenaAllocatable<'thir, 'tcx, ()> for T {
+            #[inline]
+            fn allocate_on(self, arena: &'thir Arena<'thir, 'tcx>) -> &'thir mut Self {
+                arena.dropless.alloc(self)
+            }
+            #[inline]
+            fn allocate_from_iter(
+                arena: &'thir Arena<'thir, 'tcx>,
+                iter: impl ::std::iter::IntoIterator<Item = Self>,
+            ) -> &'thir mut [Self] {
+                arena.dropless.alloc_from_iter(iter)
+            }
+
+        }
+        $(
+            impl<'thir, 'tcx> ArenaAllocatable<'thir, 'tcx, $ty> for $ty {
+                #[inline]
+                fn allocate_on(self, arena: &'thir Arena<'thir, 'tcx>) -> &'thir mut Self {
+                    if !::std::mem::needs_drop::<Self>() {
+                        return arena.dropless.alloc(self);
+                    }
+                    match rustc_arena::which_arena_for_type!($a[&arena.$name]) {
+                        ::std::option::Option::<&rustc_arena::TypedArena<Self>>::Some(ty_arena) => {
+                            ty_arena.alloc(self)
+                        }
+                        ::std::option::Option::None => unsafe { arena.drop.alloc(self) },
+                    }
+                }
+
+                #[inline]
+                fn allocate_from_iter(
+                    arena: &'thir Arena<'thir, 'tcx>,
+                    iter: impl ::std::iter::IntoIterator<Item = Self>,
+                ) -> &'thir mut [Self] {
+                    if !::std::mem::needs_drop::<Self>() {
+                        return arena.dropless.alloc_from_iter(iter);
+                    }
+                    match rustc_arena::which_arena_for_type!($a[&arena.$name]) {
+                        ::std::option::Option::<&rustc_arena::TypedArena<Self>>::Some(ty_arena) => {
+                            ty_arena.alloc_from_iter(iter)
+                        }
+                        ::std::option::Option::None => unsafe { arena.drop.alloc_from_iter(iter) },
+                    }
+                }
+            }
+        )*
+
+        impl<'thir, 'tcx> Arena<'thir, 'tcx> {
+            #[inline]
+            pub fn alloc<T: ArenaAllocatable<'thir, 'tcx, U>, U>(&'thir self, value: T) -> &'thir mut T {
+                value.allocate_on(self)
+            }
+
+            #[allow(dead_code)] // function is never used
+            #[inline]
+            pub fn alloc_slice<T: ::std::marker::Copy>(&'thir self, value: &[T]) -> &'thir mut [T] {
+                if value.is_empty() {
+                    return &mut [];
+                }
+                self.dropless.alloc_slice(value)
+            }
+
+            pub fn alloc_from_iter<T: ArenaAllocatable<'thir, 'tcx, U>, U>(
+                &'thir self,
+                iter: impl ::std::iter::IntoIterator<Item = T>,
+            ) -> &'thir mut [T] {
+                T::allocate_from_iter(self, iter)
+            }
+        }
+    }
+}
+
+declare_arena!([], [
+    [] arm: Arm<'thir, 'tcx>,
+    [] expr: Expr<'thir, 'tcx>,
+    [] field_expr: FieldExpr<'thir, 'tcx>,
+    [few] inline_asm_operand: InlineAsmOperand<'thir, 'tcx>,
+    [] stmt: Stmt<'thir, 'tcx>,
+]);
diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs
index 969f7d1e3a4..ac93d042970 100644
--- a/compiler/rustc_mir_build/src/thir/constant.rs
+++ b/compiler/rustc_mir_build/src/thir/constant.rs
@@ -1,3 +1,4 @@
+use rustc_apfloat::Float;
 use rustc_ast as ast;
 use rustc_middle::mir::interpret::{
     Allocation, ConstValue, LitToConstError, LitToConstInput, Scalar,
@@ -61,20 +62,40 @@ fn parse_float<'tcx>(num: Symbol, fty: ty::FloatTy, neg: bool) -> Result<ConstVa
     use rustc_apfloat::ieee::{Double, Single};
     let scalar = match fty {
         ty::FloatTy::F32 => {
-            num.parse::<f32>().map_err(|_| ())?;
+            let rust_f = num.parse::<f32>().map_err(|_| ())?;
             let mut f = num.parse::<Single>().unwrap_or_else(|e| {
                 panic!("apfloat::ieee::Single failed to parse `{}`: {:?}", num, e)
             });
+            assert!(
+                u128::from(rust_f.to_bits()) == f.to_bits(),
+                "apfloat::ieee::Single gave different result for `{}`: \
+                 {}({:#x}) vs Rust's {}({:#x})",
+                rust_f,
+                f,
+                f.to_bits(),
+                Single::from_bits(rust_f.to_bits().into()),
+                rust_f.to_bits()
+            );
             if neg {
                 f = -f;
             }
             Scalar::from_f32(f)
         }
         ty::FloatTy::F64 => {
-            num.parse::<f64>().map_err(|_| ())?;
+            let rust_f = num.parse::<f64>().map_err(|_| ())?;
             let mut f = num.parse::<Double>().unwrap_or_else(|e| {
                 panic!("apfloat::ieee::Double failed to parse `{}`: {:?}", num, e)
             });
+            assert!(
+                u128::from(rust_f.to_bits()) == f.to_bits(),
+                "apfloat::ieee::Double gave different result for `{}`: \
+                 {}({:#x}) vs Rust's {}({:#x})",
+                rust_f,
+                f,
+                f.to_bits(),
+                Double::from_bits(rust_f.to_bits().into()),
+                rust_f.to_bits()
+            );
             if neg {
                 f = -f;
             }
diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
index 980888df7fe..d450f8a265d 100644
--- a/compiler/rustc_mir_build/src/thir/cx/block.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -1,4 +1,3 @@
-use crate::thir::cx::to_ref::ToRef;
 use crate::thir::cx::Cx;
 use crate::thir::{self, *};
 
@@ -8,110 +7,95 @@ use rustc_middle::ty;
 
 use rustc_index::vec::Idx;
 
-impl<'tcx> Mirror<'tcx> for &'tcx hir::Block<'tcx> {
-    type Output = Block<'tcx>;
-
-    fn make_mirror(self, cx: &mut Cx<'_, 'tcx>) -> Block<'tcx> {
+impl<'thir, 'tcx> Cx<'thir, 'tcx> {
+    crate fn mirror_block(&mut self, block: &'tcx hir::Block<'tcx>) -> Block<'thir, 'tcx> {
         // We have to eagerly lower the "spine" of the statements
         // in order to get the lexical scoping correctly.
-        let stmts = mirror_stmts(cx, self.hir_id.local_id, &*self.stmts);
+        let stmts = self.mirror_stmts(block.hir_id.local_id, block.stmts);
         let opt_destruction_scope =
-            cx.region_scope_tree.opt_destruction_scope(self.hir_id.local_id);
+            self.region_scope_tree.opt_destruction_scope(block.hir_id.local_id);
         Block {
-            targeted_by_break: self.targeted_by_break,
-            region_scope: region::Scope { id: self.hir_id.local_id, data: region::ScopeData::Node },
+            targeted_by_break: block.targeted_by_break,
+            region_scope: region::Scope {
+                id: block.hir_id.local_id,
+                data: region::ScopeData::Node,
+            },
             opt_destruction_scope,
-            span: self.span,
+            span: block.span,
             stmts,
-            expr: self.expr.to_ref(),
-            safety_mode: match self.rules {
+            expr: block.expr.map(|expr| self.mirror_expr(expr)),
+            safety_mode: match block.rules {
                 hir::BlockCheckMode::DefaultBlock => BlockSafety::Safe,
-                hir::BlockCheckMode::UnsafeBlock(..) => BlockSafety::ExplicitUnsafe(self.hir_id),
+                hir::BlockCheckMode::UnsafeBlock(..) => BlockSafety::ExplicitUnsafe(block.hir_id),
                 hir::BlockCheckMode::PushUnsafeBlock(..) => BlockSafety::PushUnsafe,
                 hir::BlockCheckMode::PopUnsafeBlock(..) => BlockSafety::PopUnsafe,
             },
         }
     }
-}
 
-fn mirror_stmts<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    block_id: hir::ItemLocalId,
-    stmts: &'tcx [hir::Stmt<'tcx>],
-) -> Vec<StmtRef<'tcx>> {
-    let mut result = vec![];
-    for (index, stmt) in stmts.iter().enumerate() {
-        let hir_id = stmt.hir_id;
-        let opt_dxn_ext = cx.region_scope_tree.opt_destruction_scope(hir_id.local_id);
-        match stmt.kind {
-            hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => {
-                result.push(StmtRef::Mirror(Box::new(Stmt {
+    fn mirror_stmts(
+        &mut self,
+        block_id: hir::ItemLocalId,
+        stmts: &'tcx [hir::Stmt<'tcx>],
+    ) -> &'thir [Stmt<'thir, 'tcx>] {
+        self.arena.alloc_from_iter(stmts.iter().enumerate().filter_map(|(index, stmt)| {
+            let hir_id = stmt.hir_id;
+            let opt_dxn_ext = self.region_scope_tree.opt_destruction_scope(hir_id.local_id);
+            match stmt.kind {
+                hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => Some(Stmt {
                     kind: StmtKind::Expr {
                         scope: region::Scope { id: hir_id.local_id, data: region::ScopeData::Node },
-                        expr: expr.to_ref(),
+                        expr: self.mirror_expr(expr),
                     },
                     opt_destruction_scope: opt_dxn_ext,
-                })))
-            }
-            hir::StmtKind::Item(..) => {
-                // ignore for purposes of the MIR
-            }
-            hir::StmtKind::Local(ref local) => {
-                let remainder_scope = region::Scope {
-                    id: block_id,
-                    data: region::ScopeData::Remainder(region::FirstStatementIndex::new(index)),
-                };
+                }),
+                hir::StmtKind::Item(..) => {
+                    // ignore for purposes of the MIR
+                    None
+                }
+                hir::StmtKind::Local(ref local) => {
+                    let remainder_scope = region::Scope {
+                        id: block_id,
+                        data: region::ScopeData::Remainder(region::FirstStatementIndex::new(index)),
+                    };
 
-                let mut pattern = cx.pattern_from_hir(&local.pat);
+                    let mut pattern = self.pattern_from_hir(local.pat);
 
-                if let Some(ty) = &local.ty {
-                    if let Some(&user_ty) = cx.typeck_results.user_provided_types().get(ty.hir_id) {
-                        debug!("mirror_stmts: user_ty={:?}", user_ty);
-                        pattern = Pat {
-                            ty: pattern.ty,
-                            span: pattern.span,
-                            kind: Box::new(PatKind::AscribeUserType {
-                                ascription: thir::pattern::Ascription {
-                                    user_ty: PatTyProj::from_user_type(user_ty),
-                                    user_ty_span: ty.span,
-                                    variance: ty::Variance::Covariant,
-                                },
-                                subpattern: pattern,
-                            }),
-                        };
+                    if let Some(ty) = &local.ty {
+                        if let Some(&user_ty) =
+                            self.typeck_results.user_provided_types().get(ty.hir_id)
+                        {
+                            debug!("mirror_stmts: user_ty={:?}", user_ty);
+                            pattern = Pat {
+                                ty: pattern.ty,
+                                span: pattern.span,
+                                kind: Box::new(PatKind::AscribeUserType {
+                                    ascription: thir::pattern::Ascription {
+                                        user_ty: PatTyProj::from_user_type(user_ty),
+                                        user_ty_span: ty.span,
+                                        variance: ty::Variance::Covariant,
+                                    },
+                                    subpattern: pattern,
+                                }),
+                            };
+                        }
                     }
-                }
 
-                result.push(StmtRef::Mirror(Box::new(Stmt {
-                    kind: StmtKind::Let {
-                        remainder_scope,
-                        init_scope: region::Scope {
-                            id: hir_id.local_id,
-                            data: region::ScopeData::Node,
+                    Some(Stmt {
+                        kind: StmtKind::Let {
+                            remainder_scope,
+                            init_scope: region::Scope {
+                                id: hir_id.local_id,
+                                data: region::ScopeData::Node,
+                            },
+                            pattern,
+                            initializer: local.init.map(|init| self.mirror_expr(init)),
+                            lint_level: LintLevel::Explicit(local.hir_id),
                         },
-                        pattern,
-                        initializer: local.init.to_ref(),
-                        lint_level: LintLevel::Explicit(local.hir_id),
-                    },
-                    opt_destruction_scope: opt_dxn_ext,
-                })));
+                        opt_destruction_scope: opt_dxn_ext,
+                    })
+                }
             }
-        }
+        }))
     }
-    result
-}
-
-crate fn to_expr_ref<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    block: &'tcx hir::Block<'tcx>,
-) -> ExprRef<'tcx> {
-    let block_ty = cx.typeck_results().node_type(block.hir_id);
-    let temp_lifetime = cx.region_scope_tree.temporary_scope(block.hir_id.local_id);
-    let expr = Expr {
-        ty: block_ty,
-        temp_lifetime,
-        span: block.span,
-        kind: ExprKind::Block { body: block },
-    };
-    expr.to_ref()
 }
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 620ce360e7d..924278e1a7f 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -1,11 +1,11 @@
-use crate::thir::cx::block;
-use crate::thir::cx::to_ref::ToRef;
 use crate::thir::cx::Cx;
 use crate::thir::util::UserAnnotatedTyHelpers;
 use crate::thir::*;
+use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
 use rustc_index::vec::Idx;
+use rustc_middle::hir::place::Place as HirPlace;
 use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
 use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
 use rustc_middle::mir::interpret::Scalar;
@@ -17,45 +17,71 @@ use rustc_middle::ty::subst::{InternalSubsts, SubstsRef};
 use rustc_middle::ty::{self, AdtKind, Ty};
 use rustc_span::Span;
 
-impl<'tcx> Mirror<'tcx> for &'tcx hir::Expr<'tcx> {
-    type Output = Expr<'tcx>;
+use std::iter;
 
-    fn make_mirror(self, cx: &mut Cx<'_, 'tcx>) -> Expr<'tcx> {
-        let temp_lifetime = cx.region_scope_tree.temporary_scope(self.hir_id.local_id);
-        let expr_scope = region::Scope { id: self.hir_id.local_id, data: region::ScopeData::Node };
+impl<'thir, 'tcx> Cx<'thir, 'tcx> {
+    /// Mirrors and allocates a single [`hir::Expr`]. If you need to mirror a whole slice
+    /// of expressions, prefer using [`mirror_exprs`].
+    ///
+    /// [`mirror_exprs`]: Self::mirror_exprs
+    crate fn mirror_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) -> &'thir Expr<'thir, 'tcx> {
+        // `mirror_expr` is recursing very deep. Make sure the stack doesn't overflow.
+        ensure_sufficient_stack(|| self.arena.alloc(self.mirror_expr_inner(expr)))
+    }
+
+    /// Mirrors and allocates a slice of [`hir::Expr`]s. They will be allocated as a
+    /// contiguous sequence in memory.
+    crate fn mirror_exprs(&mut self, exprs: &'tcx [hir::Expr<'tcx>]) -> &'thir [Expr<'thir, 'tcx>] {
+        self.arena.alloc_from_iter(exprs.iter().map(|expr| self.mirror_expr_inner(expr)))
+    }
 
-        debug!("Expr::make_mirror(): id={}, span={:?}", self.hir_id, self.span);
+    /// Mirrors a [`hir::Expr`] without allocating it into the arena.
+    /// This is a separate, private function so that [`mirror_expr`] and [`mirror_exprs`] can
+    /// decide how to allocate this expression (alone or within a slice).
+    ///
+    /// [`mirror_expr`]: Self::mirror_expr
+    /// [`mirror_exprs`]: Self::mirror_exprs
+    pub(super) fn mirror_expr_inner(
+        &mut self,
+        hir_expr: &'tcx hir::Expr<'tcx>,
+    ) -> Expr<'thir, 'tcx> {
+        let temp_lifetime = self.region_scope_tree.temporary_scope(hir_expr.hir_id.local_id);
+        let expr_scope =
+            region::Scope { id: hir_expr.hir_id.local_id, data: region::ScopeData::Node };
 
-        let mut expr = make_mirror_unadjusted(cx, self);
+        debug!("Expr::make_mirror(): id={}, span={:?}", hir_expr.hir_id, hir_expr.span);
+
+        let mut expr = self.make_mirror_unadjusted(hir_expr);
 
         // Now apply adjustments, if any.
-        for adjustment in cx.typeck_results().expr_adjustments(self) {
+        for adjustment in self.typeck_results.expr_adjustments(hir_expr) {
             debug!("make_mirror: expr={:?} applying adjustment={:?}", expr, adjustment);
-            expr = apply_adjustment(cx, self, expr, adjustment);
+            expr = self.apply_adjustment(hir_expr, expr, adjustment);
         }
 
         // Next, wrap this up in the expr's scope.
         expr = Expr {
             temp_lifetime,
             ty: expr.ty,
-            span: self.span,
+            span: hir_expr.span,
             kind: ExprKind::Scope {
                 region_scope: expr_scope,
-                value: expr.to_ref(),
-                lint_level: LintLevel::Explicit(self.hir_id),
+                value: self.arena.alloc(expr),
+                lint_level: LintLevel::Explicit(hir_expr.hir_id),
             },
         };
 
         // Finally, create a destruction scope, if any.
-        if let Some(region_scope) = cx.region_scope_tree.opt_destruction_scope(self.hir_id.local_id)
+        if let Some(region_scope) =
+            self.region_scope_tree.opt_destruction_scope(hir_expr.hir_id.local_id)
         {
             expr = Expr {
                 temp_lifetime,
                 ty: expr.ty,
-                span: self.span,
+                span: hir_expr.span,
                 kind: ExprKind::Scope {
                     region_scope,
-                    value: expr.to_ref(),
+                    value: self.arena.alloc(expr),
                     lint_level: LintLevel::Inherited,
                 },
             };
@@ -64,364 +90,407 @@ impl<'tcx> Mirror<'tcx> for &'tcx hir::Expr<'tcx> {
         // OK, all done!
         expr
     }
-}
 
-fn apply_adjustment<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    hir_expr: &'tcx hir::Expr<'tcx>,
-    mut expr: Expr<'tcx>,
-    adjustment: &Adjustment<'tcx>,
-) -> Expr<'tcx> {
-    let Expr { temp_lifetime, mut span, .. } = expr;
-
-    // Adjust the span from the block, to the last expression of the
-    // block. This is a better span when returning a mutable reference
-    // with too short a lifetime. The error message will use the span
-    // from the assignment to the return place, which should only point
-    // at the returned value, not the entire function body.
-    //
-    // fn return_short_lived<'a>(x: &'a mut i32) -> &'static mut i32 {
-    //      x
-    //   // ^ error message points at this expression.
-    // }
-    let mut adjust_span = |expr: &mut Expr<'tcx>| {
-        if let ExprKind::Block { body } = expr.kind {
-            if let Some(ref last_expr) = body.expr {
-                span = last_expr.span;
-                expr.span = span;
+    fn apply_adjustment(
+        &mut self,
+        hir_expr: &'tcx hir::Expr<'tcx>,
+        mut expr: Expr<'thir, 'tcx>,
+        adjustment: &Adjustment<'tcx>,
+    ) -> Expr<'thir, 'tcx> {
+        let Expr { temp_lifetime, mut span, .. } = expr;
+
+        // Adjust the span from the block, to the last expression of the
+        // block. This is a better span when returning a mutable reference
+        // with too short a lifetime. The error message will use the span
+        // from the assignment to the return place, which should only point
+        // at the returned value, not the entire function body.
+        //
+        // fn return_short_lived<'a>(x: &'a mut i32) -> &'static mut i32 {
+        //      x
+        //   // ^ error message points at this expression.
+        // }
+        let mut adjust_span = |expr: &mut Expr<'thir, 'tcx>| {
+            if let ExprKind::Block { body } = &expr.kind {
+                if let Some(ref last_expr) = body.expr {
+                    span = last_expr.span;
+                    expr.span = span;
+                }
             }
-        }
-    };
+        };
 
-    let kind = match adjustment.kind {
-        Adjust::Pointer(PointerCast::Unsize) => {
-            adjust_span(&mut expr);
-            ExprKind::Pointer { cast: PointerCast::Unsize, source: expr.to_ref() }
-        }
-        Adjust::Pointer(cast) => ExprKind::Pointer { cast, source: expr.to_ref() },
-        Adjust::NeverToAny => ExprKind::NeverToAny { source: expr.to_ref() },
-        Adjust::Deref(None) => {
-            adjust_span(&mut expr);
-            ExprKind::Deref { arg: expr.to_ref() }
-        }
-        Adjust::Deref(Some(deref)) => {
-            // We don't need to do call adjust_span here since
-            // deref coercions always start with a built-in deref.
-            let call = deref.method_call(cx.tcx(), expr.ty);
+        let kind = match adjustment.kind {
+            Adjust::Pointer(PointerCast::Unsize) => {
+                adjust_span(&mut expr);
+                ExprKind::Pointer { cast: PointerCast::Unsize, source: self.arena.alloc(expr) }
+            }
+            Adjust::Pointer(cast) => ExprKind::Pointer { cast, source: self.arena.alloc(expr) },
+            Adjust::NeverToAny => ExprKind::NeverToAny { source: self.arena.alloc(expr) },
+            Adjust::Deref(None) => {
+                adjust_span(&mut expr);
+                ExprKind::Deref { arg: self.arena.alloc(expr) }
+            }
+            Adjust::Deref(Some(deref)) => {
+                // We don't need to do call adjust_span here since
+                // deref coercions always start with a built-in deref.
+                let call = deref.method_call(self.tcx(), expr.ty);
 
-            expr = Expr {
-                temp_lifetime,
-                ty: cx.tcx.mk_ref(deref.region, ty::TypeAndMut { ty: expr.ty, mutbl: deref.mutbl }),
-                span,
-                kind: ExprKind::Borrow {
-                    borrow_kind: deref.mutbl.to_borrow_kind(),
-                    arg: expr.to_ref(),
-                },
-            };
+                expr = Expr {
+                    temp_lifetime,
+                    ty: self
+                        .tcx
+                        .mk_ref(deref.region, ty::TypeAndMut { ty: expr.ty, mutbl: deref.mutbl }),
+                    span,
+                    kind: ExprKind::Borrow {
+                        borrow_kind: deref.mutbl.to_borrow_kind(),
+                        arg: self.arena.alloc(expr),
+                    },
+                };
 
-            overloaded_place(
-                cx,
-                hir_expr,
-                adjustment.target,
-                Some(call),
-                vec![expr.to_ref()],
-                deref.span,
-            )
-        }
-        Adjust::Borrow(AutoBorrow::Ref(_, m)) => {
-            ExprKind::Borrow { borrow_kind: m.to_borrow_kind(), arg: expr.to_ref() }
-        }
-        Adjust::Borrow(AutoBorrow::RawPtr(mutability)) => {
-            ExprKind::AddressOf { mutability, arg: expr.to_ref() }
-        }
-    };
+                self.overloaded_place(
+                    hir_expr,
+                    adjustment.target,
+                    Some(call),
+                    self.arena.alloc_from_iter(iter::once(expr)),
+                    deref.span,
+                )
+            }
+            Adjust::Borrow(AutoBorrow::Ref(_, m)) => {
+                ExprKind::Borrow { borrow_kind: m.to_borrow_kind(), arg: self.arena.alloc(expr) }
+            }
+            Adjust::Borrow(AutoBorrow::RawPtr(mutability)) => {
+                ExprKind::AddressOf { mutability, arg: self.arena.alloc(expr) }
+            }
+        };
 
-    Expr { temp_lifetime, ty: adjustment.target, span, kind }
-}
+        Expr { temp_lifetime, ty: adjustment.target, span, kind }
+    }
 
-fn make_mirror_unadjusted<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    expr: &'tcx hir::Expr<'tcx>,
-) -> Expr<'tcx> {
-    let expr_ty = cx.typeck_results().expr_ty(expr);
-    let temp_lifetime = cx.region_scope_tree.temporary_scope(expr.hir_id.local_id);
-
-    let kind = match expr.kind {
-        // Here comes the interesting stuff:
-        hir::ExprKind::MethodCall(_, method_span, ref args, fn_span) => {
-            // Rewrite a.b(c) into UFCS form like Trait::b(a, c)
-            let expr = method_callee(cx, expr, method_span, None);
-            let args = args.iter().map(|e| e.to_ref()).collect();
-            ExprKind::Call { ty: expr.ty, fun: expr.to_ref(), args, from_hir_call: true, fn_span }
-        }
+    fn make_mirror_unadjusted(&mut self, expr: &'tcx hir::Expr<'tcx>) -> Expr<'thir, 'tcx> {
+        let expr_ty = self.typeck_results().expr_ty(expr);
+        let temp_lifetime = self.region_scope_tree.temporary_scope(expr.hir_id.local_id);
 
-        hir::ExprKind::Call(ref fun, ref args) => {
-            if cx.typeck_results().is_method_call(expr) {
-                // The callee is something implementing Fn, FnMut, or FnOnce.
-                // Find the actual method implementation being called and
-                // build the appropriate UFCS call expression with the
-                // callee-object as expr parameter.
+        let kind = match expr.kind {
+            // Here comes the interesting stuff:
+            hir::ExprKind::MethodCall(_, method_span, ref args, fn_span) => {
+                // Rewrite a.b(c) into UFCS form like Trait::b(a, c)
+                let expr = self.method_callee(expr, method_span, None);
+                let args = self.mirror_exprs(args);
+                ExprKind::Call {
+                    ty: expr.ty,
+                    fun: self.arena.alloc(expr),
+                    args,
+                    from_hir_call: true,
+                    fn_span,
+                }
+            }
 
-                // rewrite f(u, v) into FnOnce::call_once(f, (u, v))
+            hir::ExprKind::Call(ref fun, ref args) => {
+                if self.typeck_results().is_method_call(expr) {
+                    // The callee is something implementing Fn, FnMut, or FnOnce.
+                    // Find the actual method implementation being called and
+                    // build the appropriate UFCS call expression with the
+                    // callee-object as expr parameter.
 
-                let method = method_callee(cx, expr, fun.span, None);
+                    // rewrite f(u, v) into FnOnce::call_once(f, (u, v))
 
-                let arg_tys = args.iter().map(|e| cx.typeck_results().expr_ty_adjusted(e));
-                let tupled_args = Expr {
-                    ty: cx.tcx.mk_tup(arg_tys),
-                    temp_lifetime,
-                    span: expr.span,
-                    kind: ExprKind::Tuple { fields: args.iter().map(ToRef::to_ref).collect() },
-                };
+                    let method = self.method_callee(expr, fun.span, None);
 
-                ExprKind::Call {
-                    ty: method.ty,
-                    fun: method.to_ref(),
-                    args: vec![fun.to_ref(), tupled_args.to_ref()],
-                    from_hir_call: true,
-                    fn_span: expr.span,
-                }
-            } else {
-                let adt_data =
-                    if let hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) = fun.kind {
-                        // Tuple-like ADTs are represented as ExprKind::Call. We convert them here.
-                        expr_ty.ty_adt_def().and_then(|adt_def| match path.res {
-                            Res::Def(DefKind::Ctor(_, CtorKind::Fn), ctor_id) => {
-                                Some((adt_def, adt_def.variant_index_with_ctor_id(ctor_id)))
-                            }
-                            Res::SelfCtor(..) => Some((adt_def, VariantIdx::new(0))),
-                            _ => None,
-                        })
-                    } else {
-                        None
+                    let arg_tys = args.iter().map(|e| self.typeck_results().expr_ty_adjusted(e));
+                    let tupled_args = Expr {
+                        ty: self.tcx.mk_tup(arg_tys),
+                        temp_lifetime,
+                        span: expr.span,
+                        kind: ExprKind::Tuple { fields: self.mirror_exprs(args) },
                     };
-                if let Some((adt_def, index)) = adt_data {
-                    let substs = cx.typeck_results().node_substs(fun.hir_id);
-                    let user_provided_types = cx.typeck_results().user_provided_types();
-                    let user_ty = user_provided_types.get(fun.hir_id).copied().map(|mut u_ty| {
-                        if let UserType::TypeOf(ref mut did, _) = &mut u_ty.value {
-                            *did = adt_def.did;
-                        }
-                        u_ty
-                    });
-                    debug!("make_mirror_unadjusted: (call) user_ty={:?}", user_ty);
 
-                    let field_refs = args
-                        .iter()
-                        .enumerate()
-                        .map(|(idx, e)| FieldExprRef { name: Field::new(idx), expr: e.to_ref() })
-                        .collect();
-                    ExprKind::Adt {
-                        adt_def,
-                        substs,
-                        variant_index: index,
-                        fields: field_refs,
-                        user_ty,
-                        base: None,
-                    }
-                } else {
                     ExprKind::Call {
-                        ty: cx.typeck_results().node_type(fun.hir_id),
-                        fun: fun.to_ref(),
-                        args: args.to_ref(),
+                        ty: method.ty,
+                        fun: self.arena.alloc(method),
+                        args: self
+                            .arena
+                            .alloc_from_iter(vec![self.mirror_expr_inner(fun), tupled_args]),
                         from_hir_call: true,
                         fn_span: expr.span,
                     }
+                } else {
+                    let adt_data =
+                        if let hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) = fun.kind {
+                            // Tuple-like ADTs are represented as ExprKind::Call. We convert them here.
+                            expr_ty.ty_adt_def().and_then(|adt_def| match path.res {
+                                Res::Def(DefKind::Ctor(_, CtorKind::Fn), ctor_id) => {
+                                    Some((adt_def, adt_def.variant_index_with_ctor_id(ctor_id)))
+                                }
+                                Res::SelfCtor(..) => Some((adt_def, VariantIdx::new(0))),
+                                _ => None,
+                            })
+                        } else {
+                            None
+                        };
+                    if let Some((adt_def, index)) = adt_data {
+                        let substs = self.typeck_results().node_substs(fun.hir_id);
+                        let user_provided_types = self.typeck_results().user_provided_types();
+                        let user_ty =
+                            user_provided_types.get(fun.hir_id).copied().map(|mut u_ty| {
+                                if let UserType::TypeOf(ref mut did, _) = &mut u_ty.value {
+                                    *did = adt_def.did;
+                                }
+                                u_ty
+                            });
+                        debug!("make_mirror_unadjusted: (call) user_ty={:?}", user_ty);
+
+                        let field_refs =
+                            self.arena.alloc_from_iter(args.iter().enumerate().map(|(idx, e)| {
+                                FieldExpr { name: Field::new(idx), expr: self.mirror_expr(e) }
+                            }));
+                        ExprKind::Adt {
+                            adt_def,
+                            substs,
+                            variant_index: index,
+                            fields: field_refs,
+                            user_ty,
+                            base: None,
+                        }
+                    } else {
+                        ExprKind::Call {
+                            ty: self.typeck_results().node_type(fun.hir_id),
+                            fun: self.mirror_expr(fun),
+                            args: self.mirror_exprs(args),
+                            from_hir_call: true,
+                            fn_span: expr.span,
+                        }
+                    }
                 }
             }
-        }
 
-        hir::ExprKind::AddrOf(hir::BorrowKind::Ref, mutbl, ref arg) => {
-            ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg: arg.to_ref() }
-        }
+            hir::ExprKind::AddrOf(hir::BorrowKind::Ref, mutbl, ref arg) => {
+                ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg: self.mirror_expr(arg) }
+            }
 
-        hir::ExprKind::AddrOf(hir::BorrowKind::Raw, mutability, ref arg) => {
-            ExprKind::AddressOf { mutability, arg: arg.to_ref() }
-        }
+            hir::ExprKind::AddrOf(hir::BorrowKind::Raw, mutability, ref arg) => {
+                ExprKind::AddressOf { mutability, arg: self.mirror_expr(arg) }
+            }
 
-        hir::ExprKind::Block(ref blk, _) => ExprKind::Block { body: &blk },
+            hir::ExprKind::Block(ref blk, _) => ExprKind::Block { body: self.mirror_block(blk) },
 
-        hir::ExprKind::Assign(ref lhs, ref rhs, _) => {
-            ExprKind::Assign { lhs: lhs.to_ref(), rhs: rhs.to_ref() }
-        }
+            hir::ExprKind::Assign(ref lhs, ref rhs, _) => {
+                ExprKind::Assign { lhs: self.mirror_expr(lhs), rhs: self.mirror_expr(rhs) }
+            }
 
-        hir::ExprKind::AssignOp(op, ref lhs, ref rhs) => {
-            if cx.typeck_results().is_method_call(expr) {
-                overloaded_operator(cx, expr, vec![lhs.to_ref(), rhs.to_ref()])
-            } else {
-                ExprKind::AssignOp { op: bin_op(op.node), lhs: lhs.to_ref(), rhs: rhs.to_ref() }
+            hir::ExprKind::AssignOp(op, ref lhs, ref rhs) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let lhs = self.mirror_expr_inner(lhs);
+                    let rhs = self.mirror_expr_inner(rhs);
+                    self.overloaded_operator(expr, self.arena.alloc_from_iter(vec![lhs, rhs]))
+                } else {
+                    ExprKind::AssignOp {
+                        op: bin_op(op.node),
+                        lhs: self.mirror_expr(lhs),
+                        rhs: self.mirror_expr(rhs),
+                    }
+                }
             }
-        }
 
-        hir::ExprKind::Lit(ref lit) => ExprKind::Literal {
-            literal: cx.const_eval_literal(&lit.node, expr_ty, lit.span, false),
-            user_ty: None,
-            const_id: None,
-        },
-
-        hir::ExprKind::Binary(op, ref lhs, ref rhs) => {
-            if cx.typeck_results().is_method_call(expr) {
-                overloaded_operator(cx, expr, vec![lhs.to_ref(), rhs.to_ref()])
-            } else {
-                // FIXME overflow
-                match (op.node, cx.constness) {
-                    (hir::BinOpKind::And, _) => ExprKind::LogicalOp {
-                        op: LogicalOp::And,
-                        lhs: lhs.to_ref(),
-                        rhs: rhs.to_ref(),
-                    },
-                    (hir::BinOpKind::Or, _) => ExprKind::LogicalOp {
-                        op: LogicalOp::Or,
-                        lhs: lhs.to_ref(),
-                        rhs: rhs.to_ref(),
-                    },
+            hir::ExprKind::Lit(ref lit) => ExprKind::Literal {
+                literal: self.const_eval_literal(&lit.node, expr_ty, lit.span, false),
+                user_ty: None,
+                const_id: None,
+            },
 
-                    _ => {
-                        let op = bin_op(op.node);
-                        ExprKind::Binary { op, lhs: lhs.to_ref(), rhs: rhs.to_ref() }
+            hir::ExprKind::Binary(op, ref lhs, ref rhs) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let lhs = self.mirror_expr_inner(lhs);
+                    let rhs = self.mirror_expr_inner(rhs);
+                    self.overloaded_operator(expr, self.arena.alloc_from_iter(vec![lhs, rhs]))
+                } else {
+                    // FIXME overflow
+                    match op.node {
+                        hir::BinOpKind::And => ExprKind::LogicalOp {
+                            op: LogicalOp::And,
+                            lhs: self.mirror_expr(lhs),
+                            rhs: self.mirror_expr(rhs),
+                        },
+                        hir::BinOpKind::Or => ExprKind::LogicalOp {
+                            op: LogicalOp::Or,
+                            lhs: self.mirror_expr(lhs),
+                            rhs: self.mirror_expr(rhs),
+                        },
+
+                        _ => {
+                            let op = bin_op(op.node);
+                            ExprKind::Binary {
+                                op,
+                                lhs: self.mirror_expr(lhs),
+                                rhs: self.mirror_expr(rhs),
+                            }
+                        }
                     }
                 }
             }
-        }
 
-        hir::ExprKind::Index(ref lhs, ref index) => {
-            if cx.typeck_results().is_method_call(expr) {
-                overloaded_place(
-                    cx,
-                    expr,
-                    expr_ty,
-                    None,
-                    vec![lhs.to_ref(), index.to_ref()],
-                    expr.span,
-                )
-            } else {
-                ExprKind::Index { lhs: lhs.to_ref(), index: index.to_ref() }
+            hir::ExprKind::Index(ref lhs, ref index) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let lhs = self.mirror_expr_inner(lhs);
+                    let index = self.mirror_expr_inner(index);
+                    self.overloaded_place(
+                        expr,
+                        expr_ty,
+                        None,
+                        self.arena.alloc_from_iter(vec![lhs, index]),
+                        expr.span,
+                    )
+                } else {
+                    ExprKind::Index { lhs: self.mirror_expr(lhs), index: self.mirror_expr(index) }
+                }
             }
-        }
 
-        hir::ExprKind::Unary(hir::UnOp::Deref, ref arg) => {
-            if cx.typeck_results().is_method_call(expr) {
-                overloaded_place(cx, expr, expr_ty, None, vec![arg.to_ref()], expr.span)
-            } else {
-                ExprKind::Deref { arg: arg.to_ref() }
+            hir::ExprKind::Unary(hir::UnOp::Deref, ref arg) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let arg = self.mirror_expr_inner(arg);
+                    self.overloaded_place(
+                        expr,
+                        expr_ty,
+                        None,
+                        self.arena.alloc_from_iter(iter::once(arg)),
+                        expr.span,
+                    )
+                } else {
+                    ExprKind::Deref { arg: self.mirror_expr(arg) }
+                }
             }
-        }
 
-        hir::ExprKind::Unary(hir::UnOp::Not, ref arg) => {
-            if cx.typeck_results().is_method_call(expr) {
-                overloaded_operator(cx, expr, vec![arg.to_ref()])
-            } else {
-                ExprKind::Unary { op: UnOp::Not, arg: arg.to_ref() }
+            hir::ExprKind::Unary(hir::UnOp::Not, ref arg) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let arg = self.mirror_expr_inner(arg);
+                    self.overloaded_operator(expr, self.arena.alloc_from_iter(iter::once(arg)))
+                } else {
+                    ExprKind::Unary { op: UnOp::Not, arg: self.mirror_expr(arg) }
+                }
             }
-        }
 
-        hir::ExprKind::Unary(hir::UnOp::Neg, ref arg) => {
-            if cx.typeck_results().is_method_call(expr) {
-                overloaded_operator(cx, expr, vec![arg.to_ref()])
-            } else if let hir::ExprKind::Lit(ref lit) = arg.kind {
-                ExprKind::Literal {
-                    literal: cx.const_eval_literal(&lit.node, expr_ty, lit.span, true),
-                    user_ty: None,
-                    const_id: None,
+            hir::ExprKind::Unary(hir::UnOp::Neg, ref arg) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let arg = self.mirror_expr_inner(arg);
+                    self.overloaded_operator(expr, self.arena.alloc_from_iter(iter::once(arg)))
+                } else if let hir::ExprKind::Lit(ref lit) = arg.kind {
+                    ExprKind::Literal {
+                        literal: self.const_eval_literal(&lit.node, expr_ty, lit.span, true),
+                        user_ty: None,
+                        const_id: None,
+                    }
+                } else {
+                    ExprKind::Unary { op: UnOp::Neg, arg: self.mirror_expr(arg) }
                 }
-            } else {
-                ExprKind::Unary { op: UnOp::Neg, arg: arg.to_ref() }
             }
-        }
 
-        hir::ExprKind::Struct(ref qpath, ref fields, ref base) => match expr_ty.kind() {
-            ty::Adt(adt, substs) => match adt.adt_kind() {
-                AdtKind::Struct | AdtKind::Union => {
-                    let user_provided_types = cx.typeck_results().user_provided_types();
-                    let user_ty = user_provided_types.get(expr.hir_id).copied();
-                    debug!("make_mirror_unadjusted: (struct/union) user_ty={:?}", user_ty);
-                    ExprKind::Adt {
-                        adt_def: adt,
-                        variant_index: VariantIdx::new(0),
-                        substs,
-                        user_ty,
-                        fields: field_refs(cx, fields),
-                        base: base.as_ref().map(|base| FruInfo {
-                            base: base.to_ref(),
-                            field_types: cx.typeck_results().fru_field_types()[expr.hir_id].clone(),
-                        }),
+            hir::ExprKind::Struct(ref qpath, ref fields, ref base) => match expr_ty.kind() {
+                ty::Adt(adt, substs) => match adt.adt_kind() {
+                    AdtKind::Struct | AdtKind::Union => {
+                        let user_provided_types = self.typeck_results().user_provided_types();
+                        let user_ty = user_provided_types.get(expr.hir_id).copied();
+                        debug!("make_mirror_unadjusted: (struct/union) user_ty={:?}", user_ty);
+                        ExprKind::Adt {
+                            adt_def: adt,
+                            variant_index: VariantIdx::new(0),
+                            substs,
+                            user_ty,
+                            fields: self.field_refs(fields),
+                            base: base.as_ref().map(|base| FruInfo {
+                                base: self.mirror_expr(base),
+                                field_types: self.arena.alloc_from_iter(
+                                    self.typeck_results().fru_field_types()[expr.hir_id]
+                                        .iter()
+                                        .cloned(),
+                                ),
+                            }),
+                        }
                     }
-                }
-                AdtKind::Enum => {
-                    let res = cx.typeck_results().qpath_res(qpath, expr.hir_id);
-                    match res {
-                        Res::Def(DefKind::Variant, variant_id) => {
-                            assert!(base.is_none());
-
-                            let index = adt.variant_index_with_id(variant_id);
-                            let user_provided_types = cx.typeck_results().user_provided_types();
-                            let user_ty = user_provided_types.get(expr.hir_id).copied();
-                            debug!("make_mirror_unadjusted: (variant) user_ty={:?}", user_ty);
-                            ExprKind::Adt {
-                                adt_def: adt,
-                                variant_index: index,
-                                substs,
-                                user_ty,
-                                fields: field_refs(cx, fields),
-                                base: None,
+                    AdtKind::Enum => {
+                        let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
+                        match res {
+                            Res::Def(DefKind::Variant, variant_id) => {
+                                assert!(base.is_none());
+
+                                let index = adt.variant_index_with_id(variant_id);
+                                let user_provided_types =
+                                    self.typeck_results().user_provided_types();
+                                let user_ty = user_provided_types.get(expr.hir_id).copied();
+                                debug!("make_mirror_unadjusted: (variant) user_ty={:?}", user_ty);
+                                ExprKind::Adt {
+                                    adt_def: adt,
+                                    variant_index: index,
+                                    substs,
+                                    user_ty,
+                                    fields: self.field_refs(fields),
+                                    base: None,
+                                }
+                            }
+                            _ => {
+                                span_bug!(expr.span, "unexpected res: {:?}", res);
                             }
-                        }
-                        _ => {
-                            span_bug!(expr.span, "unexpected res: {:?}", res);
                         }
                     }
-                }
-            },
-            _ => {
-                span_bug!(expr.span, "unexpected type for struct literal: {:?}", expr_ty);
-            }
-        },
-
-        hir::ExprKind::Closure(..) => {
-            let closure_ty = cx.typeck_results().expr_ty(expr);
-            let (def_id, substs, movability) = match *closure_ty.kind() {
-                ty::Closure(def_id, substs) => (def_id, UpvarSubsts::Closure(substs), None),
-                ty::Generator(def_id, substs, movability) => {
-                    (def_id, UpvarSubsts::Generator(substs), Some(movability))
-                }
+                },
                 _ => {
-                    span_bug!(expr.span, "closure expr w/o closure type: {:?}", closure_ty);
+                    span_bug!(expr.span, "unexpected type for struct literal: {:?}", expr_ty);
                 }
-            };
+            },
 
-            let upvars = cx
-                .typeck_results()
-                .closure_min_captures_flattened(def_id)
-                .zip(substs.upvar_tys())
-                .map(|(captured_place, ty)| capture_upvar(cx, expr, captured_place, ty))
-                .collect();
-            ExprKind::Closure { closure_id: def_id, substs, upvars, movability }
-        }
+            hir::ExprKind::Closure(..) => {
+                let closure_ty = self.typeck_results().expr_ty(expr);
+                let (def_id, substs, movability) = match *closure_ty.kind() {
+                    ty::Closure(def_id, substs) => (def_id, UpvarSubsts::Closure(substs), None),
+                    ty::Generator(def_id, substs, movability) => {
+                        (def_id, UpvarSubsts::Generator(substs), Some(movability))
+                    }
+                    _ => {
+                        span_bug!(expr.span, "closure expr w/o closure type: {:?}", closure_ty);
+                    }
+                };
 
-        hir::ExprKind::Path(ref qpath) => {
-            let res = cx.typeck_results().qpath_res(qpath, expr.hir_id);
-            convert_path_expr(cx, expr, res)
-        }
+                let upvars = self.arena.alloc_from_iter(
+                    self.typeck_results
+                        .closure_min_captures_flattened(def_id)
+                        .zip(substs.upvar_tys())
+                        .map(|(captured_place, ty)| self.capture_upvar(expr, captured_place, ty)),
+                );
+
+                // Convert the closure fake reads, if any, from hir `Place` to ExprRef
+                let fake_reads = match self.typeck_results.closure_fake_reads.get(&def_id) {
+                    Some(fake_reads) => fake_reads
+                        .iter()
+                        .map(|(place, cause, hir_id)| {
+                            let expr = self.convert_captured_hir_place(expr, place.clone());
+                            let expr_ref: &'thir Expr<'thir, 'tcx> = self.arena.alloc(expr);
+                            (expr_ref, *cause, *hir_id)
+                        })
+                        .collect(),
+                    None => Vec::new(),
+                };
+
+                ExprKind::Closure { closure_id: def_id, substs, upvars, movability, fake_reads }
+            }
+
+            hir::ExprKind::Path(ref qpath) => {
+                let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
+                self.convert_path_expr(expr, res)
+            }
 
-        hir::ExprKind::InlineAsm(ref asm) => ExprKind::InlineAsm {
-            template: asm.template,
-            operands: asm
-                .operands
-                .iter()
-                .map(|(op, _op_sp)| {
+            hir::ExprKind::InlineAsm(ref asm) => ExprKind::InlineAsm {
+                template: asm.template,
+                operands: self.arena.alloc_from_iter(asm.operands.iter().map(|(op, _op_sp)| {
                     match *op {
                         hir::InlineAsmOperand::In { reg, ref expr } => {
-                            InlineAsmOperand::In { reg, expr: expr.to_ref() }
+                            InlineAsmOperand::In { reg, expr: self.mirror_expr(expr) }
                         }
                         hir::InlineAsmOperand::Out { reg, late, ref expr } => {
                             InlineAsmOperand::Out {
                                 reg,
                                 late,
-                                expr: expr.as_ref().map(|expr| expr.to_ref()),
+                                expr: expr.as_ref().map(|expr| self.mirror_expr(expr)),
                             }
                         }
                         hir::InlineAsmOperand::InOut { reg, late, ref expr } => {
-                            InlineAsmOperand::InOut { reg, late, expr: expr.to_ref() }
+                            InlineAsmOperand::InOut { reg, late, expr: self.mirror_expr(expr) }
                         }
                         hir::InlineAsmOperand::SplitInOut {
                             reg,
@@ -431,11 +500,15 @@ fn make_mirror_unadjusted<'a, 'tcx>(
                         } => InlineAsmOperand::SplitInOut {
                             reg,
                             late,
-                            in_expr: in_expr.to_ref(),
-                            out_expr: out_expr.as_ref().map(|expr| expr.to_ref()),
+                            in_expr: self.mirror_expr(in_expr),
+                            out_expr: out_expr.as_ref().map(|expr| self.mirror_expr(expr)),
                         },
-                        hir::InlineAsmOperand::Const { ref expr } => {
-                            InlineAsmOperand::Const { expr: expr.to_ref() }
+                        hir::InlineAsmOperand::Const { ref anon_const } => {
+                            let anon_const_def_id = self.tcx.hir().local_def_id(anon_const.hir_id);
+                            let value = ty::Const::from_anon_const(self.tcx, anon_const_def_id);
+                            let span = self.tcx.hir().span(anon_const.hir_id);
+
+                            InlineAsmOperand::Const { value, span }
                         }
                         hir::InlineAsmOperand::Sym { ref expr } => {
                             let qpath = match expr.kind {
@@ -447,25 +520,24 @@ fn make_mirror_unadjusted<'a, 'tcx>(
                                 ),
                             };
                             let temp_lifetime =
-                                cx.region_scope_tree.temporary_scope(expr.hir_id.local_id);
-                            let res = cx.typeck_results().qpath_res(qpath, expr.hir_id);
+                                self.region_scope_tree.temporary_scope(expr.hir_id.local_id);
+                            let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
                             let ty;
                             match res {
                                 Res::Def(DefKind::Fn, _) | Res::Def(DefKind::AssocFn, _) => {
-                                    ty = cx.typeck_results().node_type(expr.hir_id);
-                                    let user_ty = user_substs_applied_to_res(cx, expr.hir_id, res);
+                                    ty = self.typeck_results().node_type(expr.hir_id);
+                                    let user_ty = self.user_substs_applied_to_res(expr.hir_id, res);
                                     InlineAsmOperand::SymFn {
-                                        expr: Expr {
+                                        expr: self.arena.alloc(Expr {
                                             ty,
                                             temp_lifetime,
                                             span: expr.span,
                                             kind: ExprKind::Literal {
-                                                literal: ty::Const::zero_sized(cx.tcx, ty),
+                                                literal: ty::Const::zero_sized(self.tcx, ty),
                                                 user_ty,
                                                 const_id: None,
                                             },
-                                        }
-                                        .to_ref(),
+                                        }),
                                     }
                                 }
 
@@ -474,277 +546,583 @@ fn make_mirror_unadjusted<'a, 'tcx>(
                                 }
 
                                 _ => {
-                                    cx.tcx.sess.span_err(
+                                    self.tcx.sess.span_err(
                                         expr.span,
                                         "asm `sym` operand must point to a fn or static",
                                     );
 
                                     // Not a real fn, but we're not reaching codegen anyways...
-                                    ty = cx.tcx.ty_error();
+                                    ty = self.tcx.ty_error();
                                     InlineAsmOperand::SymFn {
-                                        expr: Expr {
+                                        expr: self.arena.alloc(Expr {
                                             ty,
                                             temp_lifetime,
                                             span: expr.span,
                                             kind: ExprKind::Literal {
-                                                literal: ty::Const::zero_sized(cx.tcx, ty),
+                                                literal: ty::Const::zero_sized(self.tcx, ty),
                                                 user_ty: None,
                                                 const_id: None,
                                             },
-                                        }
-                                        .to_ref(),
+                                        }),
                                     }
                                 }
                             }
                         }
                     }
-                })
-                .collect(),
-            options: asm.options,
-            line_spans: asm.line_spans,
-        },
-
-        hir::ExprKind::LlvmInlineAsm(ref asm) => ExprKind::LlvmInlineAsm {
-            asm: &asm.inner,
-            outputs: asm.outputs_exprs.to_ref(),
-            inputs: asm.inputs_exprs.to_ref(),
-        },
-
-        hir::ExprKind::ConstBlock(ref anon_const) => {
-            let anon_const_def_id = cx.tcx.hir().local_def_id(anon_const.hir_id);
-            let value = ty::Const::from_anon_const(cx.tcx, anon_const_def_id);
-
-            ExprKind::ConstBlock { value }
-        }
-        // Now comes the rote stuff:
-        hir::ExprKind::Repeat(ref v, ref count) => {
-            let count_def_id = cx.tcx.hir().local_def_id(count.hir_id);
-            let count = ty::Const::from_anon_const(cx.tcx, count_def_id);
+                })),
+                options: asm.options,
+                line_spans: asm.line_spans,
+            },
 
-            ExprKind::Repeat { value: v.to_ref(), count }
-        }
-        hir::ExprKind::Ret(ref v) => ExprKind::Return { value: v.to_ref() },
-        hir::ExprKind::Break(dest, ref value) => match dest.target_id {
-            Ok(target_id) => ExprKind::Break {
-                label: region::Scope { id: target_id.local_id, data: region::ScopeData::Node },
-                value: value.to_ref(),
+            hir::ExprKind::LlvmInlineAsm(ref asm) => ExprKind::LlvmInlineAsm {
+                asm: &asm.inner,
+                outputs: self.mirror_exprs(asm.outputs_exprs),
+                inputs: self.mirror_exprs(asm.inputs_exprs),
+            },
+
+            hir::ExprKind::ConstBlock(ref anon_const) => {
+                let anon_const_def_id = self.tcx.hir().local_def_id(anon_const.hir_id);
+                let value = ty::Const::from_anon_const(self.tcx, anon_const_def_id);
+
+                ExprKind::ConstBlock { value }
+            }
+            // Now comes the rote stuff:
+            hir::ExprKind::Repeat(ref v, ref count) => {
+                let count_def_id = self.tcx.hir().local_def_id(count.hir_id);
+                let count = ty::Const::from_anon_const(self.tcx, count_def_id);
+
+                ExprKind::Repeat { value: self.mirror_expr(v), count }
+            }
+            hir::ExprKind::Ret(ref v) => {
+                ExprKind::Return { value: v.as_ref().map(|v| self.mirror_expr(v)) }
+            }
+            hir::ExprKind::Break(dest, ref value) => match dest.target_id {
+                Ok(target_id) => ExprKind::Break {
+                    label: region::Scope { id: target_id.local_id, data: region::ScopeData::Node },
+                    value: value.as_ref().map(|value| self.mirror_expr(value)),
+                },
+                Err(err) => bug!("invalid loop id for break: {}", err),
             },
-            Err(err) => bug!("invalid loop id for break: {}", err),
-        },
-        hir::ExprKind::Continue(dest) => match dest.target_id {
-            Ok(loop_id) => ExprKind::Continue {
-                label: region::Scope { id: loop_id.local_id, data: region::ScopeData::Node },
+            hir::ExprKind::Continue(dest) => match dest.target_id {
+                Ok(loop_id) => ExprKind::Continue {
+                    label: region::Scope { id: loop_id.local_id, data: region::ScopeData::Node },
+                },
+                Err(err) => bug!("invalid loop id for continue: {}", err),
             },
-            Err(err) => bug!("invalid loop id for continue: {}", err),
-        },
-        hir::ExprKind::If(cond, then, else_opt) => ExprKind::If {
-            cond: cond.to_ref(),
-            then: then.to_ref(),
-            else_opt: else_opt.map(|el| el.to_ref()),
-        },
-        hir::ExprKind::Match(ref discr, ref arms, _) => ExprKind::Match {
-            scrutinee: discr.to_ref(),
-            arms: arms.iter().map(|a| convert_arm(cx, a)).collect(),
-        },
-        hir::ExprKind::Loop(ref body, ..) => ExprKind::Loop { body: block::to_expr_ref(cx, body) },
-        hir::ExprKind::Field(ref source, ..) => ExprKind::Field {
-            lhs: source.to_ref(),
-            name: Field::new(cx.tcx.field_index(expr.hir_id, cx.typeck_results)),
-        },
-        hir::ExprKind::Cast(ref source, ref cast_ty) => {
-            // Check for a user-given type annotation on this `cast`
-            let user_provided_types = cx.typeck_results.user_provided_types();
-            let user_ty = user_provided_types.get(cast_ty.hir_id);
-
-            debug!(
-                "cast({:?}) has ty w/ hir_id {:?} and user provided ty {:?}",
-                expr, cast_ty.hir_id, user_ty,
-            );
-
-            // Check to see if this cast is a "coercion cast", where the cast is actually done
-            // using a coercion (or is a no-op).
-            let cast = if cx.typeck_results().is_coercion_cast(source.hir_id) {
-                // Convert the lexpr to a vexpr.
-                ExprKind::Use { source: source.to_ref() }
-            } else if cx.typeck_results().expr_ty(source).is_region_ptr() {
-                // Special cased so that we can type check that the element
-                // type of the source matches the pointed to type of the
-                // destination.
-                ExprKind::Pointer { source: source.to_ref(), cast: PointerCast::ArrayToPointer }
-            } else {
-                // check whether this is casting an enum variant discriminant
-                // to prevent cycles, we refer to the discriminant initializer
-                // which is always an integer and thus doesn't need to know the
-                // enum's layout (or its tag type) to compute it during const eval
-                // Example:
-                // enum Foo {
-                //     A,
-                //     B = A as isize + 4,
-                // }
-                // The correct solution would be to add symbolic computations to miri,
-                // so we wouldn't have to compute and store the actual value
-                let var = if let hir::ExprKind::Path(ref qpath) = source.kind {
-                    let res = cx.typeck_results().qpath_res(qpath, source.hir_id);
-                    cx.typeck_results().node_type(source.hir_id).ty_adt_def().and_then(|adt_def| {
-                        match res {
-                            Res::Def(
-                                DefKind::Ctor(CtorOf::Variant, CtorKind::Const),
-                                variant_ctor_id,
-                            ) => {
-                                let idx = adt_def.variant_index_with_ctor_id(variant_ctor_id);
-                                let (d, o) = adt_def.discriminant_def_for_variant(idx);
-                                use rustc_middle::ty::util::IntTypeExt;
-                                let ty = adt_def.repr.discr_type();
-                                let ty = ty.to_ty(cx.tcx());
-                                Some((d, o, ty))
-                            }
-                            _ => None,
-                        }
-                    })
+            hir::ExprKind::If(cond, then, else_opt) => ExprKind::If {
+                cond: self.mirror_expr(cond),
+                then: self.mirror_expr(then),
+                else_opt: else_opt.map(|el| self.mirror_expr(el)),
+            },
+            hir::ExprKind::Match(ref discr, ref arms, _) => ExprKind::Match {
+                scrutinee: self.mirror_expr(discr),
+                arms: self.arena.alloc_from_iter(arms.iter().map(|a| self.convert_arm(a))),
+            },
+            hir::ExprKind::Loop(ref body, ..) => {
+                let block_ty = self.typeck_results().node_type(body.hir_id);
+                let temp_lifetime = self.region_scope_tree.temporary_scope(body.hir_id.local_id);
+                let block = self.mirror_block(body);
+                let body = self.arena.alloc(Expr {
+                    ty: block_ty,
+                    temp_lifetime,
+                    span: block.span,
+                    kind: ExprKind::Block { body: block },
+                });
+                ExprKind::Loop { body }
+            }
+            hir::ExprKind::Field(ref source, ..) => ExprKind::Field {
+                lhs: self.mirror_expr(source),
+                name: Field::new(self.tcx.field_index(expr.hir_id, self.typeck_results)),
+            },
+            hir::ExprKind::Cast(ref source, ref cast_ty) => {
+                // Check for a user-given type annotation on this `cast`
+                let user_provided_types = self.typeck_results.user_provided_types();
+                let user_ty = user_provided_types.get(cast_ty.hir_id);
+
+                debug!(
+                    "cast({:?}) has ty w/ hir_id {:?} and user provided ty {:?}",
+                    expr, cast_ty.hir_id, user_ty,
+                );
+
+                // Check to see if this cast is a "coercion cast", where the cast is actually done
+                // using a coercion (or is a no-op).
+                let cast = if self.typeck_results().is_coercion_cast(source.hir_id) {
+                    // Convert the lexpr to a vexpr.
+                    ExprKind::Use { source: self.mirror_expr(source) }
+                } else if self.typeck_results().expr_ty(source).is_region_ptr() {
+                    // Special cased so that we can type check that the element
+                    // type of the source matches the pointed to type of the
+                    // destination.
+                    ExprKind::Pointer {
+                        source: self.mirror_expr(source),
+                        cast: PointerCast::ArrayToPointer,
+                    }
                 } else {
-                    None
-                };
-
-                let source = if let Some((did, offset, var_ty)) = var {
-                    let mk_const = |literal| {
-                        Expr {
-                            temp_lifetime,
-                            ty: var_ty,
-                            span: expr.span,
-                            kind: ExprKind::Literal { literal, user_ty: None, const_id: None },
-                        }
-                        .to_ref()
+                    // check whether this is casting an enum variant discriminant
+                    // to prevent cycles, we refer to the discriminant initializer
+                    // which is always an integer and thus doesn't need to know the
+                    // enum's layout (or its tag type) to compute it during const eval
+                    // Example:
+                    // enum Foo {
+                    //     A,
+                    //     B = A as isize + 4,
+                    // }
+                    // The correct solution would be to add symbolic computations to miri,
+                    // so we wouldn't have to compute and store the actual value
+                    let var = if let hir::ExprKind::Path(ref qpath) = source.kind {
+                        let res = self.typeck_results().qpath_res(qpath, source.hir_id);
+                        self.typeck_results().node_type(source.hir_id).ty_adt_def().and_then(
+                            |adt_def| match res {
+                                Res::Def(
+                                    DefKind::Ctor(CtorOf::Variant, CtorKind::Const),
+                                    variant_ctor_id,
+                                ) => {
+                                    let idx = adt_def.variant_index_with_ctor_id(variant_ctor_id);
+                                    let (d, o) = adt_def.discriminant_def_for_variant(idx);
+                                    use rustc_middle::ty::util::IntTypeExt;
+                                    let ty = adt_def.repr.discr_type();
+                                    let ty = ty.to_ty(self.tcx());
+                                    Some((d, o, ty))
+                                }
+                                _ => None,
+                            },
+                        )
+                    } else {
+                        None
                     };
-                    let offset = mk_const(ty::Const::from_bits(
-                        cx.tcx,
-                        offset as u128,
-                        cx.param_env.and(var_ty),
-                    ));
-                    match did {
-                        Some(did) => {
-                            // in case we are offsetting from a computed discriminant
-                            // and not the beginning of discriminants (which is always `0`)
-                            let substs = InternalSubsts::identity_for_item(cx.tcx(), did);
-                            let lhs = mk_const(cx.tcx().mk_const(ty::Const {
-                                val: ty::ConstKind::Unevaluated(
-                                    ty::WithOptConstParam::unknown(did),
-                                    substs,
-                                    None,
-                                ),
+
+                    let source = if let Some((did, offset, var_ty)) = var {
+                        let mk_const = |literal| {
+                            self.arena.alloc(Expr {
+                                temp_lifetime,
                                 ty: var_ty,
-                            }));
-                            let bin = ExprKind::Binary { op: BinOp::Add, lhs, rhs: offset };
-                            Expr { temp_lifetime, ty: var_ty, span: expr.span, kind: bin }.to_ref()
+                                span: expr.span,
+                                kind: ExprKind::Literal { literal, user_ty: None, const_id: None },
+                            })
+                        };
+                        let offset = mk_const(ty::Const::from_bits(
+                            self.tcx,
+                            offset as u128,
+                            self.param_env.and(var_ty),
+                        ));
+                        match did {
+                            Some(did) => {
+                                // in case we are offsetting from a computed discriminant
+                                // and not the beginning of discriminants (which is always `0`)
+                                let substs = InternalSubsts::identity_for_item(self.tcx(), did);
+                                let lhs = mk_const(self.tcx().mk_const(ty::Const {
+                                    val: ty::ConstKind::Unevaluated(ty::Unevaluated {
+                                        def: ty::WithOptConstParam::unknown(did),
+                                        substs,
+                                        promoted: None,
+                                    }),
+                                    ty: var_ty,
+                                }));
+                                let bin =
+                                    ExprKind::Binary { op: BinOp::Add, lhs: lhs, rhs: offset };
+                                self.arena.alloc(Expr {
+                                    temp_lifetime,
+                                    ty: var_ty,
+                                    span: expr.span,
+                                    kind: bin,
+                                })
+                            }
+                            None => offset,
                         }
-                        None => offset,
-                    }
-                } else {
-                    source.to_ref()
-                };
+                    } else {
+                        self.mirror_expr(source)
+                    };
 
-                ExprKind::Cast { source }
-            };
+                    ExprKind::Cast { source: source }
+                };
 
-            if let Some(user_ty) = user_ty {
-                // NOTE: Creating a new Expr and wrapping a Cast inside of it may be
-                //       inefficient, revisit this when performance becomes an issue.
-                let cast_expr = Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind: cast };
-                debug!("make_mirror_unadjusted: (cast) user_ty={:?}", user_ty);
+                if let Some(user_ty) = user_ty {
+                    // NOTE: Creating a new Expr and wrapping a Cast inside of it may be
+                    //       inefficient, revisit this when performance becomes an issue.
+                    let cast_expr = self.arena.alloc(Expr {
+                        temp_lifetime,
+                        ty: expr_ty,
+                        span: expr.span,
+                        kind: cast,
+                    });
+                    debug!("make_mirror_unadjusted: (cast) user_ty={:?}", user_ty);
 
-                ExprKind::ValueTypeAscription {
-                    source: cast_expr.to_ref(),
-                    user_ty: Some(*user_ty),
+                    ExprKind::ValueTypeAscription { source: cast_expr, user_ty: Some(*user_ty) }
+                } else {
+                    cast
                 }
-            } else {
-                cast
             }
-        }
-        hir::ExprKind::Type(ref source, ref ty) => {
-            let user_provided_types = cx.typeck_results.user_provided_types();
-            let user_ty = user_provided_types.get(ty.hir_id).copied();
-            debug!("make_mirror_unadjusted: (type) user_ty={:?}", user_ty);
-            if source.is_syntactic_place_expr() {
-                ExprKind::PlaceTypeAscription { source: source.to_ref(), user_ty }
-            } else {
-                ExprKind::ValueTypeAscription { source: source.to_ref(), user_ty }
+            hir::ExprKind::Type(ref source, ref ty) => {
+                let user_provided_types = self.typeck_results.user_provided_types();
+                let user_ty = user_provided_types.get(ty.hir_id).copied();
+                debug!("make_mirror_unadjusted: (type) user_ty={:?}", user_ty);
+                let mirrored = self.mirror_expr(source);
+                if source.is_syntactic_place_expr() {
+                    ExprKind::PlaceTypeAscription { source: mirrored, user_ty }
+                } else {
+                    ExprKind::ValueTypeAscription { source: mirrored, user_ty }
+                }
+            }
+            hir::ExprKind::DropTemps(ref source) => {
+                ExprKind::Use { source: self.mirror_expr(source) }
             }
+            hir::ExprKind::Box(ref value) => ExprKind::Box { value: self.mirror_expr(value) },
+            hir::ExprKind::Array(ref fields) => {
+                ExprKind::Array { fields: self.mirror_exprs(fields) }
+            }
+            hir::ExprKind::Tup(ref fields) => ExprKind::Tuple { fields: self.mirror_exprs(fields) },
+
+            hir::ExprKind::Yield(ref v, _) => ExprKind::Yield { value: self.mirror_expr(v) },
+            hir::ExprKind::Err => unreachable!(),
+        };
+
+        Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind }
+    }
+
+    fn user_substs_applied_to_res(
+        &mut self,
+        hir_id: hir::HirId,
+        res: Res,
+    ) -> Option<ty::CanonicalUserType<'tcx>> {
+        debug!("user_substs_applied_to_res: res={:?}", res);
+        let user_provided_type = match res {
+            // A reference to something callable -- e.g., a fn, method, or
+            // a tuple-struct or tuple-variant. This has the type of a
+            // `Fn` but with the user-given substitutions.
+            Res::Def(DefKind::Fn, _)
+            | Res::Def(DefKind::AssocFn, _)
+            | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
+            | Res::Def(DefKind::Const, _)
+            | Res::Def(DefKind::AssocConst, _) => {
+                self.typeck_results().user_provided_types().get(hir_id).copied()
+            }
+
+            // A unit struct/variant which is used as a value (e.g.,
+            // `None`). This has the type of the enum/struct that defines
+            // this variant -- but with the substitutions given by the
+            // user.
+            Res::Def(DefKind::Ctor(_, CtorKind::Const), _) => {
+                self.user_substs_applied_to_ty_of_hir_id(hir_id)
+            }
+
+            // `Self` is used in expression as a tuple struct constructor or an unit struct constructor
+            Res::SelfCtor(_) => self.user_substs_applied_to_ty_of_hir_id(hir_id),
+
+            _ => bug!("user_substs_applied_to_res: unexpected res {:?} at {:?}", res, hir_id),
+        };
+        debug!("user_substs_applied_to_res: user_provided_type={:?}", user_provided_type);
+        user_provided_type
+    }
+
+    fn method_callee(
+        &mut self,
+        expr: &hir::Expr<'_>,
+        span: Span,
+        overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>,
+    ) -> Expr<'thir, 'tcx> {
+        let temp_lifetime = self.region_scope_tree.temporary_scope(expr.hir_id.local_id);
+        let (def_id, substs, user_ty) = match overloaded_callee {
+            Some((def_id, substs)) => (def_id, substs, None),
+            None => {
+                let (kind, def_id) =
+                    self.typeck_results().type_dependent_def(expr.hir_id).unwrap_or_else(|| {
+                        span_bug!(expr.span, "no type-dependent def for method callee")
+                    });
+                let user_ty = self.user_substs_applied_to_res(expr.hir_id, Res::Def(kind, def_id));
+                debug!("method_callee: user_ty={:?}", user_ty);
+                (def_id, self.typeck_results().node_substs(expr.hir_id), user_ty)
+            }
+        };
+        let ty = self.tcx().mk_fn_def(def_id, substs);
+        Expr {
+            temp_lifetime,
+            ty,
+            span,
+            kind: ExprKind::Literal {
+                literal: ty::Const::zero_sized(self.tcx(), ty),
+                user_ty,
+                const_id: None,
+            },
         }
-        hir::ExprKind::DropTemps(ref source) => ExprKind::Use { source: source.to_ref() },
-        hir::ExprKind::Box(ref value) => ExprKind::Box { value: value.to_ref() },
-        hir::ExprKind::Array(ref fields) => ExprKind::Array { fields: fields.to_ref() },
-        hir::ExprKind::Tup(ref fields) => ExprKind::Tuple { fields: fields.to_ref() },
+    }
 
-        hir::ExprKind::Yield(ref v, _) => ExprKind::Yield { value: v.to_ref() },
-        hir::ExprKind::Err => unreachable!(),
-    };
+    fn convert_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) -> Arm<'thir, 'tcx> {
+        Arm {
+            pattern: self.pattern_from_hir(&arm.pat),
+            guard: arm.guard.as_ref().map(|g| match g {
+                hir::Guard::If(ref e) => Guard::If(self.mirror_expr(e)),
+                hir::Guard::IfLet(ref pat, ref e) => {
+                    Guard::IfLet(self.pattern_from_hir(pat), self.mirror_expr(e))
+                }
+            }),
+            body: self.mirror_expr(arm.body),
+            lint_level: LintLevel::Explicit(arm.hir_id),
+            scope: region::Scope { id: arm.hir_id.local_id, data: region::ScopeData::Node },
+            span: arm.span,
+        }
+    }
 
-    Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind }
-}
+    fn convert_path_expr(
+        &mut self,
+        expr: &'tcx hir::Expr<'tcx>,
+        res: Res,
+    ) -> ExprKind<'thir, 'tcx> {
+        let substs = self.typeck_results().node_substs(expr.hir_id);
+        match res {
+            // A regular function, constructor function or a constant.
+            Res::Def(DefKind::Fn, _)
+            | Res::Def(DefKind::AssocFn, _)
+            | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
+            | Res::SelfCtor(..) => {
+                let user_ty = self.user_substs_applied_to_res(expr.hir_id, res);
+                debug!("convert_path_expr: user_ty={:?}", user_ty);
+                ExprKind::Literal {
+                    literal: ty::Const::zero_sized(
+                        self.tcx,
+                        self.typeck_results().node_type(expr.hir_id),
+                    ),
+                    user_ty,
+                    const_id: None,
+                }
+            }
+
+            Res::Def(DefKind::ConstParam, def_id) => {
+                let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+                let item_id = self.tcx.hir().get_parent_node(hir_id);
+                let item_def_id = self.tcx.hir().local_def_id(item_id);
+                let generics = self.tcx.generics_of(item_def_id);
+                let index = generics.param_def_id_to_index[&def_id];
+                let name = self.tcx.hir().name(hir_id);
+                let val = ty::ConstKind::Param(ty::ParamConst::new(index, name));
+                ExprKind::Literal {
+                    literal: self.tcx.mk_const(ty::Const {
+                        val,
+                        ty: self.typeck_results().node_type(expr.hir_id),
+                    }),
+                    user_ty: None,
+                    const_id: Some(def_id),
+                }
+            }
+
+            Res::Def(DefKind::Const, def_id) | Res::Def(DefKind::AssocConst, def_id) => {
+                let user_ty = self.user_substs_applied_to_res(expr.hir_id, res);
+                debug!("convert_path_expr: (const) user_ty={:?}", user_ty);
+                ExprKind::Literal {
+                    literal: self.tcx.mk_const(ty::Const {
+                        val: ty::ConstKind::Unevaluated(ty::Unevaluated {
+                            def: ty::WithOptConstParam::unknown(def_id),
+                            substs,
+                            promoted: None,
+                        }),
+                        ty: self.typeck_results().node_type(expr.hir_id),
+                    }),
+                    user_ty,
+                    const_id: Some(def_id),
+                }
+            }
+
+            Res::Def(DefKind::Ctor(_, CtorKind::Const), def_id) => {
+                let user_provided_types = self.typeck_results.user_provided_types();
+                let user_provided_type = user_provided_types.get(expr.hir_id).copied();
+                debug!("convert_path_expr: user_provided_type={:?}", user_provided_type);
+                let ty = self.typeck_results().node_type(expr.hir_id);
+                match ty.kind() {
+                    // A unit struct/variant which is used as a value.
+                    // We return a completely different ExprKind here to account for this special case.
+                    ty::Adt(adt_def, substs) => ExprKind::Adt {
+                        adt_def,
+                        variant_index: adt_def.variant_index_with_ctor_id(def_id),
+                        substs,
+                        user_ty: user_provided_type,
+                        fields: self.arena.alloc_from_iter(iter::empty()),
+                        base: None,
+                    },
+                    _ => bug!("unexpected ty: {:?}", ty),
+                }
+            }
+
+            // We encode uses of statics as a `*&STATIC` where the `&STATIC` part is
+            // a constant reference (or constant raw pointer for `static mut`) in MIR
+            Res::Def(DefKind::Static, id) => {
+                let ty = self.tcx.static_ptr_ty(id);
+                let temp_lifetime = self.region_scope_tree.temporary_scope(expr.hir_id.local_id);
+                let kind = if self.tcx.is_thread_local_static(id) {
+                    ExprKind::ThreadLocalRef(id)
+                } else {
+                    let ptr = self.tcx.create_static_alloc(id);
+                    ExprKind::StaticRef {
+                        literal: ty::Const::from_scalar(self.tcx, Scalar::Ptr(ptr.into()), ty),
+                        def_id: id,
+                    }
+                };
+                ExprKind::Deref {
+                    arg: self.arena.alloc(Expr { ty, temp_lifetime, span: expr.span, kind }),
+                }
+            }
+
+            Res::Local(var_hir_id) => self.convert_var(var_hir_id),
 
-fn user_substs_applied_to_res<'tcx>(
-    cx: &mut Cx<'_, 'tcx>,
-    hir_id: hir::HirId,
-    res: Res,
-) -> Option<ty::CanonicalUserType<'tcx>> {
-    debug!("user_substs_applied_to_res: res={:?}", res);
-    let user_provided_type = match res {
-        // A reference to something callable -- e.g., a fn, method, or
-        // a tuple-struct or tuple-variant. This has the type of a
-        // `Fn` but with the user-given substitutions.
-        Res::Def(DefKind::Fn, _)
-        | Res::Def(DefKind::AssocFn, _)
-        | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
-        | Res::Def(DefKind::Const, _)
-        | Res::Def(DefKind::AssocConst, _) => {
-            cx.typeck_results().user_provided_types().get(hir_id).copied()
+            _ => span_bug!(expr.span, "res `{:?}` not yet implemented", res),
         }
+    }
 
-        // A unit struct/variant which is used as a value (e.g.,
-        // `None`). This has the type of the enum/struct that defines
-        // this variant -- but with the substitutions given by the
-        // user.
-        Res::Def(DefKind::Ctor(_, CtorKind::Const), _) => {
-            cx.user_substs_applied_to_ty_of_hir_id(hir_id)
+    fn convert_var(&mut self, var_hir_id: hir::HirId) -> ExprKind<'thir, 'tcx> {
+        // We want upvars here not captures.
+        // Captures will be handled in MIR.
+        let is_upvar = self
+            .tcx
+            .upvars_mentioned(self.body_owner)
+            .map_or(false, |upvars| upvars.contains_key(&var_hir_id));
+
+        debug!(
+            "convert_var({:?}): is_upvar={}, body_owner={:?}",
+            var_hir_id, is_upvar, self.body_owner
+        );
+
+        if is_upvar {
+            ExprKind::UpvarRef { closure_def_id: self.body_owner, var_hir_id }
+        } else {
+            ExprKind::VarRef { id: var_hir_id }
         }
+    }
 
-        // `Self` is used in expression as a tuple struct constructor or an unit struct constructor
-        Res::SelfCtor(_) => cx.user_substs_applied_to_ty_of_hir_id(hir_id),
+    fn overloaded_operator(
+        &mut self,
+        expr: &'tcx hir::Expr<'tcx>,
+        args: &'thir [Expr<'thir, 'tcx>],
+    ) -> ExprKind<'thir, 'tcx> {
+        let fun = self.arena.alloc(self.method_callee(expr, expr.span, None));
+        ExprKind::Call { ty: fun.ty, fun, args, from_hir_call: false, fn_span: expr.span }
+    }
 
-        _ => bug!("user_substs_applied_to_res: unexpected res {:?} at {:?}", res, hir_id),
-    };
-    debug!("user_substs_applied_to_res: user_provided_type={:?}", user_provided_type);
-    user_provided_type
-}
+    fn overloaded_place(
+        &mut self,
+        expr: &'tcx hir::Expr<'tcx>,
+        place_ty: Ty<'tcx>,
+        overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>,
+        args: &'thir [Expr<'thir, 'tcx>],
+        span: Span,
+    ) -> ExprKind<'thir, 'tcx> {
+        // For an overloaded *x or x[y] expression of type T, the method
+        // call returns an &T and we must add the deref so that the types
+        // line up (this is because `*x` and `x[y]` represent places):
+
+        // Reconstruct the output assuming it's a reference with the
+        // same region and mutability as the receiver. This holds for
+        // `Deref(Mut)::Deref(_mut)` and `Index(Mut)::index(_mut)`.
+        let (region, mutbl) = match *args[0].ty.kind() {
+            ty::Ref(region, _, mutbl) => (region, mutbl),
+            _ => span_bug!(span, "overloaded_place: receiver is not a reference"),
+        };
+        let ref_ty = self.tcx.mk_ref(region, ty::TypeAndMut { ty: place_ty, mutbl });
+
+        // construct the complete expression `foo()` for the overloaded call,
+        // which will yield the &T type
+        let temp_lifetime = self.region_scope_tree.temporary_scope(expr.hir_id.local_id);
+        let fun = self.arena.alloc(self.method_callee(expr, span, overloaded_callee));
+        let ref_expr = self.arena.alloc(Expr {
+            temp_lifetime,
+            ty: ref_ty,
+            span,
+            kind: ExprKind::Call { ty: fun.ty, fun, args, from_hir_call: false, fn_span: span },
+        });
 
-fn method_callee<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    expr: &hir::Expr<'_>,
-    span: Span,
-    overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>,
-) -> Expr<'tcx> {
-    let temp_lifetime = cx.region_scope_tree.temporary_scope(expr.hir_id.local_id);
-    let (def_id, substs, user_ty) = match overloaded_callee {
-        Some((def_id, substs)) => (def_id, substs, None),
-        None => {
-            let (kind, def_id) = cx
-                .typeck_results()
-                .type_dependent_def(expr.hir_id)
-                .unwrap_or_else(|| span_bug!(expr.span, "no type-dependent def for method callee"));
-            let user_ty = user_substs_applied_to_res(cx, expr.hir_id, Res::Def(kind, def_id));
-            debug!("method_callee: user_ty={:?}", user_ty);
-            (def_id, cx.typeck_results().node_substs(expr.hir_id), user_ty)
+        // construct and return a deref wrapper `*foo()`
+        ExprKind::Deref { arg: ref_expr }
+    }
+
+    fn convert_captured_hir_place(
+        &mut self,
+        closure_expr: &'tcx hir::Expr<'tcx>,
+        place: HirPlace<'tcx>,
+    ) -> Expr<'thir, 'tcx> {
+        let temp_lifetime = self.region_scope_tree.temporary_scope(closure_expr.hir_id.local_id);
+        let var_ty = place.base_ty;
+
+        // The result of capture analysis in `rustc_typeck/check/upvar.rs`represents a captured path
+        // as it's seen for use within the closure and not at the time of closure creation.
+        //
+        // That is we see expect to see it start from a captured upvar and not something that is local
+        // to the closure's parent.
+        let var_hir_id = match place.base {
+            HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
+            base => bug!("Expected an upvar, found {:?}", base),
+        };
+
+        let mut captured_place_expr = Expr {
+            temp_lifetime,
+            ty: var_ty,
+            span: closure_expr.span,
+            kind: self.convert_var(var_hir_id),
+        };
+
+        for proj in place.projections.iter() {
+            let kind = match proj.kind {
+                HirProjectionKind::Deref => {
+                    ExprKind::Deref { arg: self.arena.alloc(captured_place_expr) }
+                }
+                HirProjectionKind::Field(field, ..) => {
+                    // Variant index will always be 0, because for multi-variant
+                    // enums, we capture the enum entirely.
+                    ExprKind::Field {
+                        lhs: self.arena.alloc(captured_place_expr),
+                        name: Field::new(field as usize),
+                    }
+                }
+                HirProjectionKind::Index | HirProjectionKind::Subslice => {
+                    // We don't capture these projections, so we can ignore them here
+                    continue;
+                }
+            };
+
+            captured_place_expr =
+                Expr { temp_lifetime, ty: proj.ty, span: closure_expr.span, kind };
+        }
+
+        captured_place_expr
+    }
+
+    fn capture_upvar(
+        &mut self,
+        closure_expr: &'tcx hir::Expr<'tcx>,
+        captured_place: &'tcx ty::CapturedPlace<'tcx>,
+        upvar_ty: Ty<'tcx>,
+    ) -> Expr<'thir, 'tcx> {
+        let upvar_capture = captured_place.info.capture_kind;
+        let captured_place_expr =
+            self.convert_captured_hir_place(closure_expr, captured_place.place.clone());
+        let temp_lifetime = self.region_scope_tree.temporary_scope(closure_expr.hir_id.local_id);
+
+        match upvar_capture {
+            ty::UpvarCapture::ByValue(_) => captured_place_expr,
+            ty::UpvarCapture::ByRef(upvar_borrow) => {
+                let borrow_kind = match upvar_borrow.kind {
+                    ty::BorrowKind::ImmBorrow => BorrowKind::Shared,
+                    ty::BorrowKind::UniqueImmBorrow => BorrowKind::Unique,
+                    ty::BorrowKind::MutBorrow => BorrowKind::Mut { allow_two_phase_borrow: false },
+                };
+                Expr {
+                    temp_lifetime,
+                    ty: upvar_ty,
+                    span: closure_expr.span,
+                    kind: ExprKind::Borrow {
+                        borrow_kind,
+                        arg: self.arena.alloc(captured_place_expr),
+                    },
+                }
+            }
         }
-    };
-    let ty = cx.tcx().mk_fn_def(def_id, substs);
-    Expr {
-        temp_lifetime,
-        ty,
-        span,
-        kind: ExprKind::Literal {
-            literal: ty::Const::zero_sized(cx.tcx(), ty),
-            user_ty,
-            const_id: None,
-        },
+    }
+
+    /// Converts a list of named fields (i.e., for struct-like struct/enum ADTs) into FieldExpr.
+    fn field_refs(
+        &mut self,
+        fields: &'tcx [hir::ExprField<'tcx>],
+    ) -> &'thir [FieldExpr<'thir, 'tcx>] {
+        self.arena.alloc_from_iter(fields.iter().map(|field| FieldExpr {
+            name: Field::new(self.tcx.field_index(field.hir_id, self.typeck_results)),
+            expr: self.mirror_expr(field.expr),
+        }))
     }
 }
 
@@ -776,135 +1154,6 @@ impl ToBorrowKind for hir::Mutability {
     }
 }
 
-fn convert_arm<'tcx>(cx: &mut Cx<'_, 'tcx>, arm: &'tcx hir::Arm<'tcx>) -> Arm<'tcx> {
-    Arm {
-        pattern: cx.pattern_from_hir(&arm.pat),
-        guard: arm.guard.as_ref().map(|g| match g {
-            hir::Guard::If(ref e) => Guard::If(e.to_ref()),
-            hir::Guard::IfLet(ref pat, ref e) => Guard::IfLet(cx.pattern_from_hir(pat), e.to_ref()),
-        }),
-        body: arm.body.to_ref(),
-        lint_level: LintLevel::Explicit(arm.hir_id),
-        scope: region::Scope { id: arm.hir_id.local_id, data: region::ScopeData::Node },
-        span: arm.span,
-    }
-}
-
-fn convert_path_expr<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    expr: &'tcx hir::Expr<'tcx>,
-    res: Res,
-) -> ExprKind<'tcx> {
-    let substs = cx.typeck_results().node_substs(expr.hir_id);
-    match res {
-        // A regular function, constructor function or a constant.
-        Res::Def(DefKind::Fn, _)
-        | Res::Def(DefKind::AssocFn, _)
-        | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
-        | Res::SelfCtor(..) => {
-            let user_ty = user_substs_applied_to_res(cx, expr.hir_id, res);
-            debug!("convert_path_expr: user_ty={:?}", user_ty);
-            ExprKind::Literal {
-                literal: ty::Const::zero_sized(cx.tcx, cx.typeck_results().node_type(expr.hir_id)),
-                user_ty,
-                const_id: None,
-            }
-        }
-
-        Res::Def(DefKind::ConstParam, def_id) => {
-            let hir_id = cx.tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
-            let item_id = cx.tcx.hir().get_parent_node(hir_id);
-            let item_def_id = cx.tcx.hir().local_def_id(item_id);
-            let generics = cx.tcx.generics_of(item_def_id);
-            let index = generics.param_def_id_to_index[&def_id];
-            let name = cx.tcx.hir().name(hir_id);
-            let val = ty::ConstKind::Param(ty::ParamConst::new(index, name));
-            ExprKind::Literal {
-                literal: cx
-                    .tcx
-                    .mk_const(ty::Const { val, ty: cx.typeck_results().node_type(expr.hir_id) }),
-                user_ty: None,
-                const_id: Some(def_id),
-            }
-        }
-
-        Res::Def(DefKind::Const, def_id) | Res::Def(DefKind::AssocConst, def_id) => {
-            let user_ty = user_substs_applied_to_res(cx, expr.hir_id, res);
-            debug!("convert_path_expr: (const) user_ty={:?}", user_ty);
-            ExprKind::Literal {
-                literal: cx.tcx.mk_const(ty::Const {
-                    val: ty::ConstKind::Unevaluated(
-                        ty::WithOptConstParam::unknown(def_id),
-                        substs,
-                        None,
-                    ),
-                    ty: cx.typeck_results().node_type(expr.hir_id),
-                }),
-                user_ty,
-                const_id: Some(def_id),
-            }
-        }
-
-        Res::Def(DefKind::Ctor(_, CtorKind::Const), def_id) => {
-            let user_provided_types = cx.typeck_results.user_provided_types();
-            let user_provided_type = user_provided_types.get(expr.hir_id).copied();
-            debug!("convert_path_expr: user_provided_type={:?}", user_provided_type);
-            let ty = cx.typeck_results().node_type(expr.hir_id);
-            match ty.kind() {
-                // A unit struct/variant which is used as a value.
-                // We return a completely different ExprKind here to account for this special case.
-                ty::Adt(adt_def, substs) => ExprKind::Adt {
-                    adt_def,
-                    variant_index: adt_def.variant_index_with_ctor_id(def_id),
-                    substs,
-                    user_ty: user_provided_type,
-                    fields: vec![],
-                    base: None,
-                },
-                _ => bug!("unexpected ty: {:?}", ty),
-            }
-        }
-
-        // We encode uses of statics as a `*&STATIC` where the `&STATIC` part is
-        // a constant reference (or constant raw pointer for `static mut`) in MIR
-        Res::Def(DefKind::Static, id) => {
-            let ty = cx.tcx.static_ptr_ty(id);
-            let temp_lifetime = cx.region_scope_tree.temporary_scope(expr.hir_id.local_id);
-            let kind = if cx.tcx.is_thread_local_static(id) {
-                ExprKind::ThreadLocalRef(id)
-            } else {
-                let ptr = cx.tcx.create_static_alloc(id);
-                ExprKind::StaticRef {
-                    literal: ty::Const::from_scalar(cx.tcx, Scalar::Ptr(ptr.into()), ty),
-                    def_id: id,
-                }
-            };
-            ExprKind::Deref { arg: Expr { ty, temp_lifetime, span: expr.span, kind }.to_ref() }
-        }
-
-        Res::Local(var_hir_id) => convert_var(cx, var_hir_id),
-
-        _ => span_bug!(expr.span, "res `{:?}` not yet implemented", res),
-    }
-}
-
-fn convert_var<'tcx>(cx: &mut Cx<'_, 'tcx>, var_hir_id: hir::HirId) -> ExprKind<'tcx> {
-    // We want upvars here not captures.
-    // Captures will be handled in MIR.
-    let is_upvar = cx
-        .tcx
-        .upvars_mentioned(cx.body_owner)
-        .map_or(false, |upvars| upvars.contains_key(&var_hir_id));
-
-    debug!("convert_var({:?}): is_upvar={}, body_owner={:?}", var_hir_id, is_upvar, cx.body_owner);
-
-    if is_upvar {
-        ExprKind::UpvarRef { closure_def_id: cx.body_owner, var_hir_id }
-    } else {
-        ExprKind::VarRef { id: var_hir_id }
-    }
-}
-
 fn bin_op(op: hir::BinOpKind) -> BinOp {
     match op {
         hir::BinOpKind::Add => BinOp::Add,
@@ -926,139 +1175,3 @@ fn bin_op(op: hir::BinOpKind) -> BinOp {
         _ => bug!("no equivalent for ast binop {:?}", op),
     }
 }
-
-fn overloaded_operator<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    expr: &'tcx hir::Expr<'tcx>,
-    args: Vec<ExprRef<'tcx>>,
-) -> ExprKind<'tcx> {
-    let fun = method_callee(cx, expr, expr.span, None);
-    ExprKind::Call { ty: fun.ty, fun: fun.to_ref(), args, from_hir_call: false, fn_span: expr.span }
-}
-
-fn overloaded_place<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    expr: &'tcx hir::Expr<'tcx>,
-    place_ty: Ty<'tcx>,
-    overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>,
-    args: Vec<ExprRef<'tcx>>,
-    span: Span,
-) -> ExprKind<'tcx> {
-    // For an overloaded *x or x[y] expression of type T, the method
-    // call returns an &T and we must add the deref so that the types
-    // line up (this is because `*x` and `x[y]` represent places):
-
-    let recv_ty = match args[0] {
-        ExprRef::Thir(e) => cx.typeck_results().expr_ty_adjusted(e),
-        ExprRef::Mirror(ref e) => e.ty,
-    };
-
-    // Reconstruct the output assuming it's a reference with the
-    // same region and mutability as the receiver. This holds for
-    // `Deref(Mut)::Deref(_mut)` and `Index(Mut)::index(_mut)`.
-    let (region, mutbl) = match *recv_ty.kind() {
-        ty::Ref(region, _, mutbl) => (region, mutbl),
-        _ => span_bug!(span, "overloaded_place: receiver is not a reference"),
-    };
-    let ref_ty = cx.tcx.mk_ref(region, ty::TypeAndMut { ty: place_ty, mutbl });
-
-    // construct the complete expression `foo()` for the overloaded call,
-    // which will yield the &T type
-    let temp_lifetime = cx.region_scope_tree.temporary_scope(expr.hir_id.local_id);
-    let fun = method_callee(cx, expr, span, overloaded_callee);
-    let ref_expr = Expr {
-        temp_lifetime,
-        ty: ref_ty,
-        span,
-        kind: ExprKind::Call {
-            ty: fun.ty,
-            fun: fun.to_ref(),
-            args,
-            from_hir_call: false,
-            fn_span: span,
-        },
-    };
-
-    // construct and return a deref wrapper `*foo()`
-    ExprKind::Deref { arg: ref_expr.to_ref() }
-}
-
-fn capture_upvar<'a, 'tcx>(
-    cx: &mut Cx<'_, 'tcx>,
-    closure_expr: &'tcx hir::Expr<'tcx>,
-    captured_place: &'a ty::CapturedPlace<'tcx>,
-    upvar_ty: Ty<'tcx>,
-) -> ExprRef<'tcx> {
-    let upvar_capture = captured_place.info.capture_kind;
-    let temp_lifetime = cx.region_scope_tree.temporary_scope(closure_expr.hir_id.local_id);
-    let var_ty = captured_place.place.base_ty;
-
-    // The result of capture analysis in `rustc_typeck/check/upvar.rs`represents a captured path
-    // as it's seen for use within the closure and not at the time of closure creation.
-    //
-    // That is we see expect to see it start from a captured upvar and not something that is local
-    // to the closure's parent.
-    let var_hir_id = match captured_place.place.base {
-        HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
-        base => bug!("Expected an upvar, found {:?}", base),
-    };
-
-    let mut captured_place_expr = Expr {
-        temp_lifetime,
-        ty: var_ty,
-        span: closure_expr.span,
-        kind: convert_var(cx, var_hir_id),
-    };
-
-    for proj in captured_place.place.projections.iter() {
-        let kind = match proj.kind {
-            HirProjectionKind::Deref => ExprKind::Deref { arg: captured_place_expr.to_ref() },
-            HirProjectionKind::Field(field, ..) => {
-                // Variant index will always be 0, because for multi-variant
-                // enums, we capture the enum entirely.
-                ExprKind::Field {
-                    lhs: captured_place_expr.to_ref(),
-                    name: Field::new(field as usize),
-                }
-            }
-            HirProjectionKind::Index | HirProjectionKind::Subslice => {
-                // We don't capture these projections, so we can ignore them here
-                continue;
-            }
-        };
-
-        captured_place_expr = Expr { temp_lifetime, ty: proj.ty, span: closure_expr.span, kind };
-    }
-
-    match upvar_capture {
-        ty::UpvarCapture::ByValue(_) => captured_place_expr.to_ref(),
-        ty::UpvarCapture::ByRef(upvar_borrow) => {
-            let borrow_kind = match upvar_borrow.kind {
-                ty::BorrowKind::ImmBorrow => BorrowKind::Shared,
-                ty::BorrowKind::UniqueImmBorrow => BorrowKind::Unique,
-                ty::BorrowKind::MutBorrow => BorrowKind::Mut { allow_two_phase_borrow: false },
-            };
-            Expr {
-                temp_lifetime,
-                ty: upvar_ty,
-                span: closure_expr.span,
-                kind: ExprKind::Borrow { borrow_kind, arg: captured_place_expr.to_ref() },
-            }
-            .to_ref()
-        }
-    }
-}
-
-/// Converts a list of named fields (i.e., for struct-like struct/enum ADTs) into FieldExprRef.
-fn field_refs<'a, 'tcx>(
-    cx: &mut Cx<'a, 'tcx>,
-    fields: &'tcx [hir::Field<'tcx>],
-) -> Vec<FieldExprRef<'tcx>> {
-    fields
-        .iter()
-        .map(|field| FieldExprRef {
-            name: Field::new(cx.tcx.field_index(field.hir_id, cx.typeck_results)),
-            expr: field.expr.to_ref(),
-        })
-        .collect()
-}
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
index 465808cea9d..c0433604f8c 100644
--- a/compiler/rustc_mir_build/src/thir/cx/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -2,6 +2,7 @@
 //! structures into the THIR. The `builder` is generally ignorant of the tcx,
 //! etc., and instead goes through the `Cx` for most of its work.
 
+use crate::thir::arena::Arena;
 use crate::thir::util::UserAnnotatedTyHelpers;
 use crate::thir::*;
 
@@ -9,118 +10,48 @@ use rustc_ast as ast;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::Node;
-use rustc_index::vec::Idx;
-use rustc_infer::infer::InferCtxt;
 use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::{LitToConstError, LitToConstInput};
-use rustc_middle::ty::subst::Subst;
-use rustc_middle::ty::subst::{GenericArg, InternalSubsts};
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::symbol::{sym, Symbol};
-use rustc_target::abi::VariantIdx;
-use rustc_trait_selection::infer::InferCtxtExt;
 
-#[derive(Clone)]
-crate struct Cx<'a, 'tcx> {
+pub fn build_thir<'thir, 'tcx>(
     tcx: TyCtxt<'tcx>,
-    infcx: &'a InferCtxt<'a, 'tcx>,
+    owner_def: ty::WithOptConstParam<LocalDefId>,
+    arena: &'thir Arena<'thir, 'tcx>,
+    expr: &'tcx hir::Expr<'tcx>,
+) -> &'thir Expr<'thir, 'tcx> {
+    Cx::new(tcx, owner_def, &arena).mirror_expr(expr)
+}
 
-    crate root_lint_level: hir::HirId,
-    crate param_env: ty::ParamEnv<'tcx>,
+struct Cx<'thir, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    arena: &'thir Arena<'thir, 'tcx>,
 
-    /// Identity `InternalSubsts` for use with const-evaluation.
-    crate identity_substs: &'tcx InternalSubsts<'tcx>,
+    crate param_env: ty::ParamEnv<'tcx>,
 
     crate region_scope_tree: &'tcx region::ScopeTree,
-    crate typeck_results: &'a ty::TypeckResults<'tcx>,
-
-    /// This is `Constness::Const` if we are compiling a `static`,
-    /// `const`, or the body of a `const fn`.
-    constness: hir::Constness,
+    crate typeck_results: &'tcx ty::TypeckResults<'tcx>,
 
     /// The `DefId` of the owner of this body.
     body_owner: DefId,
-
-    /// What kind of body is being compiled.
-    crate body_owner_kind: hir::BodyOwnerKind,
-
-    /// Whether this constant/function needs overflow checks.
-    check_overflow: bool,
 }
 
-impl<'a, 'tcx> Cx<'a, 'tcx> {
-    crate fn new(
-        infcx: &'a InferCtxt<'a, 'tcx>,
+impl<'thir, 'tcx> Cx<'thir, 'tcx> {
+    fn new(
+        tcx: TyCtxt<'tcx>,
         def: ty::WithOptConstParam<LocalDefId>,
-        src_id: hir::HirId,
-    ) -> Cx<'a, 'tcx> {
-        let tcx = infcx.tcx;
+        arena: &'thir Arena<'thir, 'tcx>,
+    ) -> Cx<'thir, 'tcx> {
         let typeck_results = tcx.typeck_opt_const_arg(def);
-        let body_owner_kind = tcx.hir().body_owner_kind(src_id);
-
-        let constness = match body_owner_kind {
-            hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_) => hir::Constness::Const,
-            hir::BodyOwnerKind::Closure | hir::BodyOwnerKind::Fn => hir::Constness::NotConst,
-        };
-
-        let attrs = tcx.hir().attrs(src_id);
-
-        // Some functions always have overflow checks enabled,
-        // however, they may not get codegen'd, depending on
-        // the settings for the crate they are codegened in.
-        let mut check_overflow = tcx.sess.contains_name(attrs, sym::rustc_inherit_overflow_checks);
-
-        // Respect -C overflow-checks.
-        check_overflow |= tcx.sess.overflow_checks();
-
-        // Constants always need overflow checks.
-        check_overflow |= constness == hir::Constness::Const;
-
         Cx {
             tcx,
-            infcx,
-            root_lint_level: src_id,
+            arena,
             param_env: tcx.param_env(def.did),
-            identity_substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
             region_scope_tree: tcx.region_scope_tree(def.did),
             typeck_results,
-            constness,
             body_owner: def.did.to_def_id(),
-            body_owner_kind,
-            check_overflow,
         }
     }
-}
-
-impl<'a, 'tcx> Cx<'a, 'tcx> {
-    /// Normalizes `ast` into the appropriate "mirror" type.
-    crate fn mirror<M: Mirror<'tcx>>(&mut self, ast: M) -> M::Output {
-        ast.make_mirror(self)
-    }
-
-    crate fn usize_ty(&mut self) -> Ty<'tcx> {
-        self.tcx.types.usize
-    }
-
-    crate fn usize_literal(&mut self, value: u64) -> &'tcx ty::Const<'tcx> {
-        ty::Const::from_usize(self.tcx, value)
-    }
-
-    crate fn bool_ty(&mut self) -> Ty<'tcx> {
-        self.tcx.types.bool
-    }
-
-    crate fn unit_ty(&mut self) -> Ty<'tcx> {
-        self.tcx.mk_unit()
-    }
-
-    crate fn true_literal(&mut self) -> &'tcx ty::Const<'tcx> {
-        ty::Const::from_bool(self.tcx, true)
-    }
-
-    crate fn false_literal(&mut self) -> &'tcx ty::Const<'tcx> {
-        ty::Const::from_bool(self.tcx, false)
-    }
 
     crate fn const_eval_literal(
         &mut self,
@@ -137,11 +68,11 @@ impl<'a, 'tcx> Cx<'a, 'tcx> {
                 // FIXME(#31407) this is only necessary because float parsing is buggy
                 self.tcx.sess.span_err(sp, "could not evaluate float literal (see issue #31407)");
                 // create a dummy value and continue compiling
-                Const::from_bits(self.tcx, 0, self.param_env.and(ty))
+                self.tcx.const_error(ty)
             }
             Err(LitToConstError::Reported) => {
                 // create a dummy value and continue compiling
-                Const::from_bits(self.tcx, 0, self.param_env.and(ty))
+                self.tcx.const_error(ty)
             }
             Err(LitToConstError::TypeError) => bug!("const_eval_literal: had type error"),
         }
@@ -154,69 +85,17 @@ impl<'a, 'tcx> Cx<'a, 'tcx> {
         };
         Pat::from_hir(self.tcx, self.param_env, self.typeck_results(), p)
     }
-
-    crate fn trait_method(
-        &mut self,
-        trait_def_id: DefId,
-        method_name: Symbol,
-        self_ty: Ty<'tcx>,
-        params: &[GenericArg<'tcx>],
-    ) -> &'tcx ty::Const<'tcx> {
-        let substs = self.tcx.mk_substs_trait(self_ty, params);
-
-        // The unhygienic comparison here is acceptable because this is only
-        // used on known traits.
-        let item = self
-            .tcx
-            .associated_items(trait_def_id)
-            .filter_by_name_unhygienic(method_name)
-            .find(|item| item.kind == ty::AssocKind::Fn)
-            .expect("trait method not found");
-
-        let method_ty = self.tcx.type_of(item.def_id);
-        let method_ty = method_ty.subst(self.tcx, substs);
-        ty::Const::zero_sized(self.tcx, method_ty)
-    }
-
-    crate fn all_fields(&mut self, adt_def: &ty::AdtDef, variant_index: VariantIdx) -> Vec<Field> {
-        (0..adt_def.variants[variant_index].fields.len()).map(Field::new).collect()
-    }
-
-    crate fn needs_drop(&mut self, ty: Ty<'tcx>) -> bool {
-        ty.needs_drop(self.tcx, self.param_env)
-    }
-
-    crate fn infcx(&self) -> &'a InferCtxt<'a, 'tcx> {
-        self.infcx
-    }
-
-    crate fn tcx(&self) -> TyCtxt<'tcx> {
-        self.tcx
-    }
-
-    crate fn typeck_results(&self) -> &'a ty::TypeckResults<'tcx> {
-        self.typeck_results
-    }
-
-    crate fn check_overflow(&self) -> bool {
-        self.check_overflow
-    }
-
-    crate fn type_is_copy_modulo_regions(&self, ty: Ty<'tcx>, span: Span) -> bool {
-        self.infcx.type_is_copy_modulo_regions(self.param_env, ty, span)
-    }
 }
 
 impl<'tcx> UserAnnotatedTyHelpers<'tcx> for Cx<'_, 'tcx> {
     fn tcx(&self) -> TyCtxt<'tcx> {
-        self.tcx()
+        self.tcx
     }
 
     fn typeck_results(&self) -> &ty::TypeckResults<'tcx> {
-        self.typeck_results()
+        self.typeck_results
     }
 }
 
 mod block;
 mod expr;
-mod to_ref;
diff --git a/compiler/rustc_mir_build/src/thir/cx/to_ref.rs b/compiler/rustc_mir_build/src/thir/cx/to_ref.rs
deleted file mode 100644
index 53a988ebb79..00000000000
--- a/compiler/rustc_mir_build/src/thir/cx/to_ref.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-use crate::thir::*;
-
-use rustc_hir as hir;
-
-crate trait ToRef {
-    type Output;
-    fn to_ref(self) -> Self::Output;
-}
-
-impl<'tcx> ToRef for &'tcx hir::Expr<'tcx> {
-    type Output = ExprRef<'tcx>;
-
-    fn to_ref(self) -> ExprRef<'tcx> {
-        ExprRef::Thir(self)
-    }
-}
-
-impl<'tcx> ToRef for &'tcx &'tcx hir::Expr<'tcx> {
-    type Output = ExprRef<'tcx>;
-
-    fn to_ref(self) -> ExprRef<'tcx> {
-        ExprRef::Thir(&**self)
-    }
-}
-
-impl<'tcx> ToRef for Expr<'tcx> {
-    type Output = ExprRef<'tcx>;
-
-    fn to_ref(self) -> ExprRef<'tcx> {
-        ExprRef::Mirror(Box::new(self))
-    }
-}
-
-impl<'tcx, T, U> ToRef for &'tcx Option<T>
-where
-    &'tcx T: ToRef<Output = U>,
-{
-    type Output = Option<U>;
-
-    fn to_ref(self) -> Option<U> {
-        self.as_ref().map(|expr| expr.to_ref())
-    }
-}
-
-impl<'tcx, T, U> ToRef for &'tcx Vec<T>
-where
-    &'tcx T: ToRef<Output = U>,
-{
-    type Output = Vec<U>;
-
-    fn to_ref(self) -> Vec<U> {
-        self.iter().map(|expr| expr.to_ref()).collect()
-    }
-}
-
-impl<'tcx, T, U> ToRef for &'tcx [T]
-where
-    &'tcx T: ToRef<Output = U>,
-{
-    type Output = Vec<U>;
-
-    fn to_ref(self) -> Vec<U> {
-        self.iter().map(|expr| expr.to_ref()).collect()
-    }
-}
diff --git a/compiler/rustc_mir_build/src/thir/mod.rs b/compiler/rustc_mir_build/src/thir/mod.rs
index ed3d3927825..f4596d523d0 100644
--- a/compiler/rustc_mir_build/src/thir/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/mod.rs
@@ -4,13 +4,12 @@
 //! unit-tested and separated from the Rust source and compiler data
 //! structures.
 
-use self::cx::Cx;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_middle::infer::canonical::Canonical;
 use rustc_middle::middle::region;
-use rustc_middle::mir::{BinOp, BorrowKind, Field, UnOp};
+use rustc_middle::mir::{BinOp, BorrowKind, FakeReadCause, Field, UnOp};
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{AdtDef, Const, Ty, UpvarSubsts, UserType};
@@ -19,58 +18,57 @@ use rustc_target::abi::VariantIdx;
 use rustc_target::asm::InlineAsmRegOrRegClass;
 
 crate mod constant;
+
 crate mod cx;
+pub use cx::build_thir;
 
 crate mod pattern;
-crate use self::pattern::PatTyProj;
-crate use self::pattern::{BindingMode, FieldPat, Pat, PatKind, PatRange};
+pub use self::pattern::{Ascription, BindingMode, FieldPat, Pat, PatKind, PatRange, PatTyProj};
+
+mod arena;
+pub use arena::Arena;
 
 mod util;
 
 #[derive(Copy, Clone, Debug)]
-crate enum LintLevel {
+pub enum LintLevel {
     Inherited,
     Explicit(hir::HirId),
 }
 
-#[derive(Clone, Debug)]
-crate struct Block<'tcx> {
-    crate targeted_by_break: bool,
-    crate region_scope: region::Scope,
-    crate opt_destruction_scope: Option<region::Scope>,
-    crate span: Span,
-    crate stmts: Vec<StmtRef<'tcx>>,
-    crate expr: Option<ExprRef<'tcx>>,
-    crate safety_mode: BlockSafety,
+#[derive(Debug)]
+pub struct Block<'thir, 'tcx> {
+    pub targeted_by_break: bool,
+    pub region_scope: region::Scope,
+    pub opt_destruction_scope: Option<region::Scope>,
+    pub span: Span,
+    pub stmts: &'thir [Stmt<'thir, 'tcx>],
+    pub expr: Option<&'thir Expr<'thir, 'tcx>>,
+    pub safety_mode: BlockSafety,
 }
 
 #[derive(Copy, Clone, Debug)]
-crate enum BlockSafety {
+pub enum BlockSafety {
     Safe,
     ExplicitUnsafe(hir::HirId),
     PushUnsafe,
     PopUnsafe,
 }
 
-#[derive(Clone, Debug)]
-crate enum StmtRef<'tcx> {
-    Mirror(Box<Stmt<'tcx>>),
-}
-
-#[derive(Clone, Debug)]
-crate struct Stmt<'tcx> {
-    crate kind: StmtKind<'tcx>,
-    crate opt_destruction_scope: Option<region::Scope>,
+#[derive(Debug)]
+pub struct Stmt<'thir, 'tcx> {
+    pub kind: StmtKind<'thir, 'tcx>,
+    pub opt_destruction_scope: Option<region::Scope>,
 }
 
-#[derive(Clone, Debug)]
-crate enum StmtKind<'tcx> {
+#[derive(Debug)]
+pub enum StmtKind<'thir, 'tcx> {
     Expr {
         /// scope for this statement; may be used as lifetime of temporaries
         scope: region::Scope,
 
         /// expression being evaluated in this statement
-        expr: ExprRef<'tcx>,
+        expr: &'thir Expr<'thir, 'tcx>,
     },
 
     Let {
@@ -88,7 +86,7 @@ crate enum StmtKind<'tcx> {
         pattern: Pat<'tcx>,
 
         /// let pat: ty = <INIT> ...
-        initializer: Option<ExprRef<'tcx>>,
+        initializer: Option<&'thir Expr<'thir, 'tcx>>,
 
         /// the lint level for this let-statement
         lint_level: LintLevel,
@@ -96,13 +94,13 @@ crate enum StmtKind<'tcx> {
 }
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Expr<'_>, 168);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+rustc_data_structures::static_assert_size!(Expr<'_, '_>, 144);
 
 /// The Thir trait implementor lowers their expressions (`&'tcx H::Expr`)
 /// into instances of this `Expr` enum. This lowering can be done
 /// basically as lazily or as eagerly as desired: every recursive
-/// reference to an expression in this enum is an `ExprRef<'tcx>`, which
+/// reference to an expression in this enum is an `&'thir Expr<'thir, 'tcx>`, which
 /// may in turn be another instance of this enum (boxed), or else an
 /// unlowered `&'tcx H::Expr`. Note that instances of `Expr` are very
 /// short-lived. They are created by `Thir::to_expr`, analyzed and
@@ -113,105 +111,105 @@ rustc_data_structures::static_assert_size!(Expr<'_>, 168);
 /// MIR simplifications are already done in the impl of `Thir`. For
 /// example, method calls and overloaded operators are absent: they are
 /// expected to be converted into `Expr::Call` instances.
-#[derive(Clone, Debug)]
-crate struct Expr<'tcx> {
+#[derive(Debug)]
+pub struct Expr<'thir, 'tcx> {
     /// type of this expression
-    crate ty: Ty<'tcx>,
+    pub ty: Ty<'tcx>,
 
     /// lifetime of this expression if it should be spilled into a
     /// temporary; should be None only if in a constant context
-    crate temp_lifetime: Option<region::Scope>,
+    pub temp_lifetime: Option<region::Scope>,
 
     /// span of the expression in the source
-    crate span: Span,
+    pub span: Span,
 
     /// kind of expression
-    crate kind: ExprKind<'tcx>,
+    pub kind: ExprKind<'thir, 'tcx>,
 }
 
-#[derive(Clone, Debug)]
-crate enum ExprKind<'tcx> {
+#[derive(Debug)]
+pub enum ExprKind<'thir, 'tcx> {
     Scope {
         region_scope: region::Scope,
         lint_level: LintLevel,
-        value: ExprRef<'tcx>,
+        value: &'thir Expr<'thir, 'tcx>,
     },
     Box {
-        value: ExprRef<'tcx>,
+        value: &'thir Expr<'thir, 'tcx>,
     },
     If {
-        cond: ExprRef<'tcx>,
-        then: ExprRef<'tcx>,
-        else_opt: Option<ExprRef<'tcx>>,
+        cond: &'thir Expr<'thir, 'tcx>,
+        then: &'thir Expr<'thir, 'tcx>,
+        else_opt: Option<&'thir Expr<'thir, 'tcx>>,
     },
     Call {
         ty: Ty<'tcx>,
-        fun: ExprRef<'tcx>,
-        args: Vec<ExprRef<'tcx>>,
-        // Whether this is from a call in HIR, rather than from an overloaded
-        // operator. True for overloaded function call.
+        fun: &'thir Expr<'thir, 'tcx>,
+        args: &'thir [Expr<'thir, 'tcx>],
+        /// Whether this is from a call in HIR, rather than from an overloaded
+        /// operator. `true` for overloaded function call.
         from_hir_call: bool,
         /// This `Span` is the span of the function, without the dot and receiver
         /// (e.g. `foo(a, b)` in `x.foo(a, b)`
         fn_span: Span,
     },
     Deref {
-        arg: ExprRef<'tcx>,
+        arg: &'thir Expr<'thir, 'tcx>,
     }, // NOT overloaded!
     Binary {
         op: BinOp,
-        lhs: ExprRef<'tcx>,
-        rhs: ExprRef<'tcx>,
+        lhs: &'thir Expr<'thir, 'tcx>,
+        rhs: &'thir Expr<'thir, 'tcx>,
     }, // NOT overloaded!
     LogicalOp {
         op: LogicalOp,
-        lhs: ExprRef<'tcx>,
-        rhs: ExprRef<'tcx>,
+        lhs: &'thir Expr<'thir, 'tcx>,
+        rhs: &'thir Expr<'thir, 'tcx>,
     }, // NOT overloaded!
     // LogicalOp is distinct from BinaryOp because of lazy evaluation of the operands.
     Unary {
         op: UnOp,
-        arg: ExprRef<'tcx>,
+        arg: &'thir Expr<'thir, 'tcx>,
     }, // NOT overloaded!
     Cast {
-        source: ExprRef<'tcx>,
+        source: &'thir Expr<'thir, 'tcx>,
     },
     Use {
-        source: ExprRef<'tcx>,
+        source: &'thir Expr<'thir, 'tcx>,
     }, // Use a lexpr to get a vexpr.
     NeverToAny {
-        source: ExprRef<'tcx>,
+        source: &'thir Expr<'thir, 'tcx>,
     },
     Pointer {
         cast: PointerCast,
-        source: ExprRef<'tcx>,
+        source: &'thir Expr<'thir, 'tcx>,
     },
     Loop {
-        body: ExprRef<'tcx>,
+        body: &'thir Expr<'thir, 'tcx>,
     },
     Match {
-        scrutinee: ExprRef<'tcx>,
-        arms: Vec<Arm<'tcx>>,
+        scrutinee: &'thir Expr<'thir, 'tcx>,
+        arms: &'thir [Arm<'thir, 'tcx>],
     },
     Block {
-        body: &'tcx hir::Block<'tcx>,
+        body: Block<'thir, 'tcx>,
     },
     Assign {
-        lhs: ExprRef<'tcx>,
-        rhs: ExprRef<'tcx>,
+        lhs: &'thir Expr<'thir, 'tcx>,
+        rhs: &'thir Expr<'thir, 'tcx>,
     },
     AssignOp {
         op: BinOp,
-        lhs: ExprRef<'tcx>,
-        rhs: ExprRef<'tcx>,
+        lhs: &'thir Expr<'thir, 'tcx>,
+        rhs: &'thir Expr<'thir, 'tcx>,
     },
     Field {
-        lhs: ExprRef<'tcx>,
+        lhs: &'thir Expr<'thir, 'tcx>,
         name: Field,
     },
     Index {
-        lhs: ExprRef<'tcx>,
-        index: ExprRef<'tcx>,
+        lhs: &'thir Expr<'thir, 'tcx>,
+        index: &'thir Expr<'thir, 'tcx>,
     },
     VarRef {
         id: hir::HirId,
@@ -226,35 +224,35 @@ crate enum ExprKind<'tcx> {
     },
     Borrow {
         borrow_kind: BorrowKind,
-        arg: ExprRef<'tcx>,
+        arg: &'thir Expr<'thir, 'tcx>,
     },
     /// A `&raw [const|mut] $place_expr` raw borrow resulting in type `*[const|mut] T`.
     AddressOf {
         mutability: hir::Mutability,
-        arg: ExprRef<'tcx>,
+        arg: &'thir Expr<'thir, 'tcx>,
     },
     Break {
         label: region::Scope,
-        value: Option<ExprRef<'tcx>>,
+        value: Option<&'thir Expr<'thir, 'tcx>>,
     },
     Continue {
         label: region::Scope,
     },
     Return {
-        value: Option<ExprRef<'tcx>>,
+        value: Option<&'thir Expr<'thir, 'tcx>>,
     },
     ConstBlock {
         value: &'tcx Const<'tcx>,
     },
     Repeat {
-        value: ExprRef<'tcx>,
+        value: &'thir Expr<'thir, 'tcx>,
         count: &'tcx Const<'tcx>,
     },
     Array {
-        fields: Vec<ExprRef<'tcx>>,
+        fields: &'thir [Expr<'thir, 'tcx>],
     },
     Tuple {
-        fields: Vec<ExprRef<'tcx>>,
+        fields: &'thir [Expr<'thir, 'tcx>],
     },
     Adt {
         adt_def: &'tcx AdtDef,
@@ -265,24 +263,25 @@ crate enum ExprKind<'tcx> {
         /// Bar::<T> { ... }`.
         user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
 
-        fields: Vec<FieldExprRef<'tcx>>,
-        base: Option<FruInfo<'tcx>>,
+        fields: &'thir [FieldExpr<'thir, 'tcx>],
+        base: Option<FruInfo<'thir, 'tcx>>,
     },
     PlaceTypeAscription {
-        source: ExprRef<'tcx>,
+        source: &'thir Expr<'thir, 'tcx>,
         /// Type that the user gave to this expression
         user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
     },
     ValueTypeAscription {
-        source: ExprRef<'tcx>,
+        source: &'thir Expr<'thir, 'tcx>,
         /// Type that the user gave to this expression
         user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
     },
     Closure {
         closure_id: DefId,
         substs: UpvarSubsts<'tcx>,
-        upvars: Vec<ExprRef<'tcx>>,
+        upvars: &'thir [Expr<'thir, 'tcx>],
         movability: Option<hir::Movability>,
+        fake_reads: Vec<(&'thir Expr<'thir, 'tcx>, FakeReadCause, hir::HirId)>,
     },
     Literal {
         literal: &'tcx Const<'tcx>,
@@ -302,7 +301,7 @@ crate enum ExprKind<'tcx> {
     },
     InlineAsm {
         template: &'tcx [InlineAsmTemplatePiece],
-        operands: Vec<InlineAsmOperand<'tcx>>,
+        operands: &'thir [InlineAsmOperand<'thir, 'tcx>],
         options: InlineAsmOptions,
         line_spans: &'tcx [Span],
     },
@@ -310,158 +309,78 @@ crate enum ExprKind<'tcx> {
     ThreadLocalRef(DefId),
     LlvmInlineAsm {
         asm: &'tcx hir::LlvmInlineAsmInner,
-        outputs: Vec<ExprRef<'tcx>>,
-        inputs: Vec<ExprRef<'tcx>>,
+        outputs: &'thir [Expr<'thir, 'tcx>],
+        inputs: &'thir [Expr<'thir, 'tcx>],
     },
     Yield {
-        value: ExprRef<'tcx>,
+        value: &'thir Expr<'thir, 'tcx>,
     },
 }
 
-#[derive(Clone, Debug)]
-crate enum ExprRef<'tcx> {
-    Thir(&'tcx hir::Expr<'tcx>),
-    Mirror(Box<Expr<'tcx>>),
+#[derive(Debug)]
+pub struct FieldExpr<'thir, 'tcx> {
+    pub name: Field,
+    pub expr: &'thir Expr<'thir, 'tcx>,
 }
 
-#[derive(Clone, Debug)]
-crate struct FieldExprRef<'tcx> {
-    crate name: Field,
-    crate expr: ExprRef<'tcx>,
+#[derive(Debug)]
+pub struct FruInfo<'thir, 'tcx> {
+    pub base: &'thir Expr<'thir, 'tcx>,
+    pub field_types: &'thir [Ty<'tcx>],
 }
 
-#[derive(Clone, Debug)]
-crate struct FruInfo<'tcx> {
-    crate base: ExprRef<'tcx>,
-    crate field_types: Vec<Ty<'tcx>>,
+#[derive(Debug)]
+pub struct Arm<'thir, 'tcx> {
+    pub pattern: Pat<'tcx>,
+    pub guard: Option<Guard<'thir, 'tcx>>,
+    pub body: &'thir Expr<'thir, 'tcx>,
+    pub lint_level: LintLevel,
+    pub scope: region::Scope,
+    pub span: Span,
 }
 
-#[derive(Clone, Debug)]
-crate struct Arm<'tcx> {
-    crate pattern: Pat<'tcx>,
-    crate guard: Option<Guard<'tcx>>,
-    crate body: ExprRef<'tcx>,
-    crate lint_level: LintLevel,
-    crate scope: region::Scope,
-    crate span: Span,
-}
-
-#[derive(Clone, Debug)]
-crate enum Guard<'tcx> {
-    If(ExprRef<'tcx>),
-    IfLet(Pat<'tcx>, ExprRef<'tcx>),
+#[derive(Debug)]
+pub enum Guard<'thir, 'tcx> {
+    If(&'thir Expr<'thir, 'tcx>),
+    IfLet(Pat<'tcx>, &'thir Expr<'thir, 'tcx>),
 }
 
 #[derive(Copy, Clone, Debug)]
-crate enum LogicalOp {
+pub enum LogicalOp {
     And,
     Or,
 }
 
-impl<'tcx> ExprRef<'tcx> {
-    crate fn span(&self) -> Span {
-        match self {
-            ExprRef::Thir(expr) => expr.span,
-            ExprRef::Mirror(expr) => expr.span,
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-crate enum InlineAsmOperand<'tcx> {
+#[derive(Debug)]
+pub enum InlineAsmOperand<'thir, 'tcx> {
     In {
         reg: InlineAsmRegOrRegClass,
-        expr: ExprRef<'tcx>,
+        expr: &'thir Expr<'thir, 'tcx>,
     },
     Out {
         reg: InlineAsmRegOrRegClass,
         late: bool,
-        expr: Option<ExprRef<'tcx>>,
+        expr: Option<&'thir Expr<'thir, 'tcx>>,
     },
     InOut {
         reg: InlineAsmRegOrRegClass,
         late: bool,
-        expr: ExprRef<'tcx>,
+        expr: &'thir Expr<'thir, 'tcx>,
     },
     SplitInOut {
         reg: InlineAsmRegOrRegClass,
         late: bool,
-        in_expr: ExprRef<'tcx>,
-        out_expr: Option<ExprRef<'tcx>>,
+        in_expr: &'thir Expr<'thir, 'tcx>,
+        out_expr: Option<&'thir Expr<'thir, 'tcx>>,
     },
     Const {
-        expr: ExprRef<'tcx>,
+        value: &'tcx Const<'tcx>,
+        span: Span,
     },
     SymFn {
-        expr: ExprRef<'tcx>,
+        expr: &'thir Expr<'thir, 'tcx>,
     },
     SymStatic {
         def_id: DefId,
     },
 }
-
-///////////////////////////////////////////////////////////////////////////
-// The Mirror trait
-
-/// "Mirroring" is the process of converting from a HIR type into one
-/// of the THIR types defined in this file. This is basically a "on
-/// the fly" desugaring step that hides a lot of the messiness in the
-/// tcx. For example, the mirror of a `&'tcx hir::Expr` is an
-/// `Expr<'tcx>`.
-///
-/// Mirroring is gradual: when you mirror an outer expression like `e1
-/// + e2`, the references to the inner expressions `e1` and `e2` are
-/// `ExprRef<'tcx>` instances, and they may or may not be eagerly
-/// mirrored. This allows a single AST node from the compiler to
-/// expand into one or more Thir nodes, which lets the Thir nodes be
-/// simpler.
-crate trait Mirror<'tcx> {
-    type Output;
-
-    fn make_mirror(self, cx: &mut Cx<'_, 'tcx>) -> Self::Output;
-}
-
-impl<'tcx> Mirror<'tcx> for Expr<'tcx> {
-    type Output = Expr<'tcx>;
-
-    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Expr<'tcx> {
-        self
-    }
-}
-
-impl<'tcx> Mirror<'tcx> for ExprRef<'tcx> {
-    type Output = Expr<'tcx>;
-
-    fn make_mirror(self, hir: &mut Cx<'_, 'tcx>) -> Expr<'tcx> {
-        match self {
-            ExprRef::Thir(h) => h.make_mirror(hir),
-            ExprRef::Mirror(m) => *m,
-        }
-    }
-}
-
-impl<'tcx> Mirror<'tcx> for Stmt<'tcx> {
-    type Output = Stmt<'tcx>;
-
-    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Stmt<'tcx> {
-        self
-    }
-}
-
-impl<'tcx> Mirror<'tcx> for StmtRef<'tcx> {
-    type Output = Stmt<'tcx>;
-
-    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Stmt<'tcx> {
-        match self {
-            StmtRef::Mirror(m) => *m,
-        }
-    }
-}
-
-impl<'tcx> Mirror<'tcx> for Block<'tcx> {
-    type Output = Block<'tcx>;
-
-    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Block<'tcx> {
-        self
-    }
-}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index ef1419b5b74..c0624c805a6 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -246,6 +246,18 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
             })
     }
 
+    fn field_pats(
+        &self,
+        vals: impl Iterator<Item = &'tcx ty::Const<'tcx>>,
+    ) -> Result<Vec<FieldPat<'tcx>>, FallbackToConstRef> {
+        vals.enumerate()
+            .map(|(idx, val)| {
+                let field = Field::new(idx);
+                Ok(FieldPat { field, pattern: self.recur(val, false)? })
+            })
+            .collect()
+    }
+
     // Recursive helper for `to_pat`; invoke that (instead of calling this directly).
     fn recur(
         &self,
@@ -257,16 +269,6 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
         let tcx = self.tcx();
         let param_env = self.param_env;
 
-        let field_pats = |vals: &[&'tcx ty::Const<'tcx>]| -> Result<_, _> {
-            vals.iter()
-                .enumerate()
-                .map(|(idx, val)| {
-                    let field = Field::new(idx);
-                    Ok(FieldPat { field, pattern: self.recur(val, false)? })
-                })
-                .collect()
-        };
-
         let kind = match cv.ty.kind() {
             ty::Float(_) => {
                 tcx.struct_span_lint_hir(
@@ -361,12 +363,12 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
                     variant_index: destructured
                         .variant
                         .expect("destructed const of adt without variant id"),
-                    subpatterns: field_pats(destructured.fields)?,
+                    subpatterns: self.field_pats(destructured.fields.iter().copied())?,
                 }
             }
             ty::Tuple(_) | ty::Adt(_, _) => {
                 let destructured = tcx.destructure_const(param_env.and(cv));
-                PatKind::Leaf { subpatterns: field_pats(destructured.fields)? }
+                PatKind::Leaf { subpatterns: self.field_pats(destructured.fields.iter().copied())? }
             }
             ty::Array(..) => PatKind::Array {
                 prefix: tcx
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 3a67eeff92c..8c740a7ec15 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -1343,7 +1343,9 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
         match &mut fields {
             Fields::Vec(pats) => {
                 for (i, pat) in new_pats {
-                    pats[i] = pat
+                    if let Some(p) = pats.get_mut(i) {
+                        *p = pat;
+                    }
                 }
             }
             Fields::Filtered { fields, .. } => {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index 6e29e60b303..9ac79a37ac6 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -40,22 +40,22 @@ crate enum PatternError {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
-crate enum BindingMode {
+pub enum BindingMode {
     ByValue,
     ByRef(BorrowKind),
 }
 
 #[derive(Clone, Debug, PartialEq)]
-crate struct FieldPat<'tcx> {
-    crate field: Field,
-    crate pattern: Pat<'tcx>,
+pub struct FieldPat<'tcx> {
+    pub field: Field,
+    pub pattern: Pat<'tcx>,
 }
 
 #[derive(Clone, Debug, PartialEq)]
-crate struct Pat<'tcx> {
-    crate ty: Ty<'tcx>,
-    crate span: Span,
-    crate kind: Box<PatKind<'tcx>>,
+pub struct Pat<'tcx> {
+    pub ty: Ty<'tcx>,
+    pub span: Span,
+    pub kind: Box<PatKind<'tcx>>,
 }
 
 impl<'tcx> Pat<'tcx> {
@@ -65,8 +65,8 @@ impl<'tcx> Pat<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
-crate struct PatTyProj<'tcx> {
-    crate user_ty: CanonicalUserType<'tcx>,
+pub struct PatTyProj<'tcx> {
+    pub user_ty: CanonicalUserType<'tcx>,
 }
 
 impl<'tcx> PatTyProj<'tcx> {
@@ -92,8 +92,8 @@ impl<'tcx> PatTyProj<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
-crate struct Ascription<'tcx> {
-    crate user_ty: PatTyProj<'tcx>,
+pub struct Ascription<'tcx> {
+    pub user_ty: PatTyProj<'tcx>,
     /// Variance to use when relating the type `user_ty` to the **type of the value being
     /// matched**. Typically, this is `Variance::Covariant`, since the value being matched must
     /// have a type that is some subtype of the ascribed type.
@@ -112,12 +112,12 @@ crate struct Ascription<'tcx> {
     /// requires that `&'static str <: T_x`, where `T_x` is the type of `x`. Really, we should
     /// probably be checking for a `PartialEq` impl instead, but this preserves the behavior
     /// of the old type-check for now. See #57280 for details.
-    crate variance: ty::Variance,
-    crate user_ty_span: Span,
+    pub variance: ty::Variance,
+    pub user_ty_span: Span,
 }
 
 #[derive(Clone, Debug, PartialEq)]
-crate enum PatKind<'tcx> {
+pub enum PatKind<'tcx> {
     Wild,
 
     AscribeUserType {
@@ -195,10 +195,10 @@ crate enum PatKind<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
-crate struct PatRange<'tcx> {
-    crate lo: &'tcx ty::Const<'tcx>,
-    crate hi: &'tcx ty::Const<'tcx>,
-    crate end: RangeEnd,
+pub struct PatRange<'tcx> {
+    pub lo: &'tcx ty::Const<'tcx>,
+    pub hi: &'tcx ty::Const<'tcx>,
+    pub end: RangeEnd,
 }
 
 impl<'tcx> fmt::Display for Pat<'tcx> {
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index 4bf870eb7ce..bd8dfd678a9 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -128,7 +128,7 @@ impl<'a> StringReader<'a> {
     }
 
     /// Turns simple `rustc_lexer::TokenKind` enum into a rich
-    /// `librustc_ast::TokenKind`. This turns strings into interned
+    /// `rustc_ast::TokenKind`. This turns strings into interned
     /// symbols and runs additional validation.
     fn cook_lexer_token(&self, token: rustc_lexer::TokenKind, start: BytePos) -> Option<TokenKind> {
         Some(match token {
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
index cea4de72df5..905077a48e2 100644
--- a/compiler/rustc_parse/src/lib.rs
+++ b/compiler/rustc_parse/src/lib.rs
@@ -1,16 +1,21 @@
 //! The main parser interface.
 
+#![feature(array_windows)]
 #![feature(crate_visibility_modifier)]
 #![feature(bindings_after_at)]
 #![feature(iter_order_by)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(box_syntax)]
 #![feature(box_patterns)]
+#![recursion_limit = "256"]
 
 use rustc_ast as ast;
-use rustc_ast::token::{self, Nonterminal};
-use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens, LazyTokenStream, TokenStream};
+use rustc_ast::token::{self, Nonterminal, Token, TokenKind};
+use rustc_ast::tokenstream::{self, AttributesData, CanSynthesizeMissingTokens, LazyTokenStream};
+use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
+use rustc_ast::tokenstream::{Spacing, TokenStream};
 use rustc_ast::AstLike;
+use rustc_ast::Attribute;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Diagnostic, FatalError, Level, PResult};
@@ -20,8 +25,6 @@ use rustc_span::{FileName, SourceFile, Span};
 use std::path::Path;
 use std::str;
 
-use tracing::debug;
-
 pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments");
 
 #[macro_use]
@@ -254,19 +257,23 @@ pub fn nt_to_tokenstream(
     // before we fall back to the stringification.
 
     let convert_tokens =
-        |tokens: Option<&LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream());
+        |tokens: Option<&LazyTokenStream>| Some(tokens?.create_token_stream().to_tokenstream());
 
     let tokens = match *nt {
-        Nonterminal::NtItem(ref item) => prepend_attrs(sess, &item.attrs, nt, item.tokens.as_ref()),
+        Nonterminal::NtItem(ref item) => prepend_attrs(&item.attrs, item.tokens.as_ref()),
         Nonterminal::NtBlock(ref block) => convert_tokens(block.tokens.as_ref()),
         Nonterminal::NtStmt(ref stmt) => {
-            let do_prepend = |tokens| prepend_attrs(sess, stmt.attrs(), nt, tokens);
             if let ast::StmtKind::Empty = stmt.kind {
-                let tokens: TokenStream =
-                    tokenstream::TokenTree::token(token::Semi, stmt.span).into();
-                do_prepend(Some(&LazyTokenStream::new(tokens)))
+                let tokens = AttrAnnotatedTokenStream::new(vec![(
+                    tokenstream::AttrAnnotatedTokenTree::Token(Token::new(
+                        TokenKind::Semi,
+                        stmt.span,
+                    )),
+                    Spacing::Alone,
+                )]);
+                prepend_attrs(&stmt.attrs(), Some(&LazyTokenStream::new(tokens)))
             } else {
-                do_prepend(stmt.tokens())
+                prepend_attrs(&stmt.attrs(), stmt.tokens())
             }
         }
         Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()),
@@ -282,10 +289,7 @@ pub fn nt_to_tokenstream(
         Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),
         Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
         Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
-            if expr.tokens.is_none() {
-                debug!("missing tokens for expr {:?}", expr);
-            }
-            prepend_attrs(sess, &expr.attrs, nt, expr.tokens.as_ref())
+            prepend_attrs(&expr.attrs, expr.tokens.as_ref())
         }
     };
 
@@ -294,34 +298,30 @@ pub fn nt_to_tokenstream(
     } else if matches!(synthesize_tokens, CanSynthesizeMissingTokens::Yes) {
         return fake_token_stream(sess, nt);
     } else {
-        panic!("Missing tokens for nt at {:?}: {:?}", nt.span(), pprust::nonterminal_to_string(nt));
+        panic!(
+            "Missing tokens for nt {:?} at {:?}: {:?}",
+            nt,
+            nt.span(),
+            pprust::nonterminal_to_string(nt)
+        );
     }
 }
 
+fn prepend_attrs(attrs: &[Attribute], tokens: Option<&LazyTokenStream>) -> Option<TokenStream> {
+    let tokens = tokens?;
+    if attrs.is_empty() {
+        return Some(tokens.create_token_stream().to_tokenstream());
+    }
+    let attr_data = AttributesData { attrs: attrs.to_vec().into(), tokens: tokens.clone() };
+    let wrapped = AttrAnnotatedTokenStream::new(vec![(
+        AttrAnnotatedTokenTree::Attributes(attr_data),
+        Spacing::Alone,
+    )]);
+    Some(wrapped.to_tokenstream())
+}
+
 pub fn fake_token_stream(sess: &ParseSess, nt: &Nonterminal) -> TokenStream {
     let source = pprust::nonterminal_to_string(nt);
     let filename = FileName::macro_expansion_source_code(&source);
     parse_stream_from_source_str(filename, source, sess, Some(nt.span()))
 }
-
-fn prepend_attrs(
-    sess: &ParseSess,
-    attrs: &[ast::Attribute],
-    nt: &Nonterminal,
-    tokens: Option<&tokenstream::LazyTokenStream>,
-) -> Option<tokenstream::TokenStream> {
-    if attrs.is_empty() {
-        return Some(tokens?.create_token_stream());
-    }
-    let mut builder = tokenstream::TokenStreamBuilder::new();
-    for attr in attrs {
-        // FIXME: Correctly handle tokens for inner attributes.
-        // For now, we fall back to reparsing the original AST node
-        if attr.style == ast::AttrStyle::Inner {
-            return Some(fake_token_stream(sess, nt));
-        }
-        builder.push(attr.tokens());
-    }
-    builder.push(tokens?.create_token_stream());
-    Some(builder.build())
-}
diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs
index 95d4a48b845..ee6ff4dba39 100644
--- a/compiler/rustc_parse/src/parser/attr.rs
+++ b/compiler/rustc_parse/src/parser/attr.rs
@@ -1,10 +1,11 @@
-use super::{AttrWrapper, Parser, PathStyle};
+use super::{AttrWrapper, Capturing, Parser, PathStyle};
 use rustc_ast as ast;
 use rustc_ast::attr;
 use rustc_ast::token::{self, Nonterminal};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{error_code, PResult};
 use rustc_span::{sym, Span};
+use std::convert::TryInto;
 
 use tracing::debug;
 
@@ -29,6 +30,7 @@ impl<'a> Parser<'a> {
     pub(super) fn parse_outer_attributes(&mut self) -> PResult<'a, AttrWrapper> {
         let mut attrs: Vec<ast::Attribute> = Vec::new();
         let mut just_parsed_doc_comment = false;
+        let start_pos = self.token_cursor.num_next_calls;
         loop {
             debug!("parse_outer_attributes: self.token={:?}", self.token);
             let attr = if self.check(&token::Pound) {
@@ -74,7 +76,7 @@ impl<'a> Parser<'a> {
                 break;
             }
         }
-        Ok(AttrWrapper::new(attrs))
+        Ok(AttrWrapper::new(attrs.into(), start_pos))
     }
 
     /// Matches `attribute = # ! [ meta_item ]`.
@@ -177,6 +179,7 @@ impl<'a> Parser<'a> {
     crate fn parse_inner_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
         let mut attrs: Vec<ast::Attribute> = vec![];
         loop {
+            let start_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap();
             // Only try to parse if it is an inner attribute (has `!`).
             let attr = if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) {
                 Some(self.parse_attribute(InnerAttrPolicy::Permitted)?)
@@ -191,6 +194,18 @@ impl<'a> Parser<'a> {
                 None
             };
             if let Some(attr) = attr {
+                let end_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap();
+                // If we are currently capturing tokens, mark the location of this inner attribute.
+                // If capturing ends up creating a `LazyTokenStream`, we will include
+                // this replace range with it, removing the inner attribute from the final
+                // `AttrAnnotatedTokenStream`. Inner attributes are stored in the parsed AST note.
+                // During macro expansion, they are selectively inserted back into the
+                // token stream (the first inner attribute is remoevd each time we invoke the
+                // corresponding macro).
+                let range = start_pos..end_pos;
+                if let Capturing::Yes = self.capture_state.capturing {
+                    self.capture_state.inner_attr_ranges.insert(attr.id, (range, vec![]));
+                }
                 attrs.push(attr);
             } else {
                 break;
@@ -311,6 +326,9 @@ pub fn maybe_needs_tokens(attrs: &[ast::Attribute]) -> bool {
     // One of the attributes may either itself be a macro,
     // or expand to macro attributes (`cfg_attr`).
     attrs.iter().any(|attr| {
+        if attr.is_doc_comment() {
+            return false;
+        }
         attr.ident().map_or(true, |ident| {
             ident.name == sym::cfg_attr || !rustc_feature::is_builtin_attr_name(ident.name)
         })
diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs
index f45d8d6c7a0..35759a396e8 100644
--- a/compiler/rustc_parse/src/parser/attr_wrapper.rs
+++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs
@@ -1,12 +1,14 @@
-use super::attr;
-use super::{ForceCollect, Parser, TokenCursor, TrailingToken};
-use rustc_ast::token::{self, Token, TokenKind};
-use rustc_ast::tokenstream::{CreateTokenStream, TokenStream, TokenTree, TreeAndSpacing};
-use rustc_ast::tokenstream::{DelimSpan, LazyTokenStream, Spacing};
-use rustc_ast::AstLike;
+use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
+use rustc_ast::token::{self, DelimToken, Token, TokenKind};
+use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttributesData, CreateTokenStream};
+use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream, Spacing};
 use rustc_ast::{self as ast};
+use rustc_ast::{AstLike, AttrVec, Attribute};
 use rustc_errors::PResult;
-use rustc_span::{Span, DUMMY_SP};
+use rustc_span::{sym, Span, DUMMY_SP};
+
+use std::convert::TryInto;
+use std::ops::Range;
 
 /// A wrapper type to ensure that the parser handles outer attributes correctly.
 /// When we parse outer attributes, we need to ensure that we capture tokens
@@ -23,23 +25,158 @@ use rustc_span::{Span, DUMMY_SP};
 /// cannot directly access the `attrs` field
 #[derive(Debug, Clone)]
 pub struct AttrWrapper {
-    attrs: Vec<ast::Attribute>,
+    attrs: AttrVec,
+    // The start of the outer attributes in the token cursor.
+    // This allows us to create a `ReplaceRange` for the entire attribute
+    // target, including outer attributes.
+    start_pos: usize,
 }
 
+// This struct is passed around very frequently,
+// so make sure it doesn't accidentally get larger
+#[cfg(target_arch = "x86_64")]
+rustc_data_structures::static_assert_size!(AttrWrapper, 16);
+
 impl AttrWrapper {
-    pub fn empty() -> AttrWrapper {
-        AttrWrapper { attrs: vec![] }
+    pub(super) fn new(attrs: AttrVec, start_pos: usize) -> AttrWrapper {
+        AttrWrapper { attrs, start_pos }
     }
-    pub fn new(attrs: Vec<ast::Attribute>) -> AttrWrapper {
-        AttrWrapper { attrs }
+    pub fn empty() -> AttrWrapper {
+        AttrWrapper { attrs: AttrVec::new(), start_pos: usize::MAX }
     }
     // FIXME: Delay span bug here?
-    pub(crate) fn take_for_recovery(self) -> Vec<ast::Attribute> {
+    pub(crate) fn take_for_recovery(self) -> AttrVec {
         self.attrs
     }
+
+    // FIXME: require passing an NT to prevent misuse of this method
+    pub(crate) fn prepend_to_nt_inner(self, attrs: &mut Vec<Attribute>) {
+        let mut self_attrs: Vec<_> = self.attrs.into();
+        std::mem::swap(attrs, &mut self_attrs);
+        attrs.extend(self_attrs);
+    }
+
     pub fn is_empty(&self) -> bool {
         self.attrs.is_empty()
     }
+
+    pub fn maybe_needs_tokens(&self) -> bool {
+        crate::parser::attr::maybe_needs_tokens(&self.attrs)
+    }
+}
+
+/// Returns `true` if `attrs` contains a `cfg` or `cfg_attr` attribute
+fn has_cfg_or_cfg_attr(attrs: &[Attribute]) -> bool {
+    // NOTE: Builtin attributes like `cfg` and `cfg_attr` cannot be renamed via imports.
+    // Therefore, the absence of a literal `cfg` or `cfg_attr` guarantees that
+    // we don't need to do any eager expansion.
+    attrs.iter().any(|attr| {
+        attr.ident().map_or(false, |ident| ident.name == sym::cfg || ident.name == sym::cfg_attr)
+    })
+}
+
+// Produces a `TokenStream` on-demand. Using `cursor_snapshot`
+// and `num_calls`, we can reconstruct the `TokenStream` seen
+// by the callback. This allows us to avoid producing a `TokenStream`
+// if it is never needed - for example, a captured `macro_rules!`
+// argument that is never passed to a proc macro.
+// In practice token stream creation happens rarely compared to
+// calls to `collect_tokens` (see some statistics in #78736),
+// so we are doing as little up-front work as possible.
+//
+// This also makes `Parser` very cheap to clone, since
+// there is no intermediate collection buffer to clone.
+#[derive(Clone)]
+struct LazyTokenStreamImpl {
+    start_token: (Token, Spacing),
+    cursor_snapshot: TokenCursor,
+    num_calls: usize,
+    break_last_token: bool,
+    replace_ranges: Box<[ReplaceRange]>,
+}
+
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 144);
+
+impl CreateTokenStream for LazyTokenStreamImpl {
+    fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
+        // The token produced by the final call to `next` or `next_desugared`
+        // was not actually consumed by the callback. The combination
+        // of chaining the initial token and using `take` produces the desired
+        // result - we produce an empty `TokenStream` if no calls were made,
+        // and omit the final token otherwise.
+        let mut cursor_snapshot = self.cursor_snapshot.clone();
+        let tokens =
+            std::iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1))
+                .chain((0..self.num_calls).map(|_| {
+                    let token = if cursor_snapshot.desugar_doc_comments {
+                        cursor_snapshot.next_desugared()
+                    } else {
+                        cursor_snapshot.next()
+                    };
+                    (FlatToken::Token(token.0), token.1)
+                }))
+                .take(self.num_calls);
+
+        if !self.replace_ranges.is_empty() {
+            let mut tokens: Vec<_> = tokens.collect();
+            let mut replace_ranges = self.replace_ranges.clone();
+            replace_ranges.sort_by_key(|(range, _)| range.start);
+
+            #[cfg(debug_assertions)]
+            {
+                for [(range, tokens), (next_range, next_tokens)] in replace_ranges.array_windows() {
+                    assert!(
+                        range.end <= next_range.start || range.end >= next_range.end,
+                        "Replace ranges should either be disjoint or nested: ({:?}, {:?}) ({:?}, {:?})",
+                        range,
+                        tokens,
+                        next_range,
+                        next_tokens,
+                    );
+                }
+            }
+
+            // Process the replace ranges, starting from the highest start
+            // position and working our way back. If have tokens like:
+            //
+            // `#[cfg(FALSE)]` struct Foo { #[cfg(FALSE)] field: bool }`
+            //
+            // Then we will generate replace ranges for both
+            // the `#[cfg(FALSE)] field: bool` and the entire
+            // `#[cfg(FALSE)]` struct Foo { #[cfg(FALSE)] field: bool }`
+            //
+            // By starting processing from the replace range with the greatest
+            // start position, we ensure that any replace range which encloses
+            // another replace range will capture the *replaced* tokens for the inner
+            // range, not the original tokens.
+            for (range, new_tokens) in replace_ranges.iter().rev() {
+                assert!(!range.is_empty(), "Cannot replace an empty range: {:?}", range);
+                // Replace ranges are only allowed to decrease the number of tokens.
+                assert!(
+                    range.len() >= new_tokens.len(),
+                    "Range {:?} has greater len than {:?}",
+                    range,
+                    new_tokens
+                );
+
+                // Replace any removed tokens with `FlatToken::Empty`.
+                // This keeps the total length of `tokens` constant throughout the
+                // replacement process, allowing us to use all of the `ReplaceRanges` entries
+                // without adjusting indices.
+                let filler = std::iter::repeat((FlatToken::Empty, Spacing::Alone))
+                    .take(range.len() - new_tokens.len());
+
+                tokens.splice(
+                    (range.start as usize)..(range.end as usize),
+                    new_tokens.clone().into_iter().chain(filler),
+                );
+            }
+            make_token_stream(tokens.into_iter(), self.break_last_token)
+        } else {
+            make_token_stream(tokens, self.break_last_token)
+        }
+    }
 }
 
 impl<'a> Parser<'a> {
@@ -65,77 +202,195 @@ impl<'a> Parser<'a> {
         force_collect: ForceCollect,
         f: impl FnOnce(&mut Self, Vec<ast::Attribute>) -> PResult<'a, (R, TrailingToken)>,
     ) -> PResult<'a, R> {
-        if matches!(force_collect, ForceCollect::No) && !attr::maybe_needs_tokens(&attrs.attrs) {
-            return Ok(f(self, attrs.attrs)?.0);
+        // We only bail out when nothing could possibly observe the collected tokens:
+        // 1. We cannot be force collecting tokens (since force-collecting requires tokens
+        //    by definition
+        if matches!(force_collect, ForceCollect::No)
+            // None of our outer attributes can require tokens (e.g. a proc-macro)
+            && !attrs.maybe_needs_tokens()
+            // If our target supports custom inner attributes, then we cannot bail
+            // out early, since we may need to capture tokens for a custom inner attribute
+            // invocation.
+            && !R::SUPPORTS_CUSTOM_INNER_ATTRS
+            // Never bail out early in `capture_cfg` mode, since there might be `#[cfg]`
+            // or `#[cfg_attr]` attributes.
+            && !self.capture_cfg
+        {
+            return Ok(f(self, attrs.attrs.into())?.0);
         }
+
         let start_token = (self.token.clone(), self.token_spacing);
         let cursor_snapshot = self.token_cursor.clone();
 
-        let (mut ret, trailing_token) = f(self, attrs.attrs)?;
-
-        // Produces a `TokenStream` on-demand. Using `cursor_snapshot`
-        // and `num_calls`, we can reconstruct the `TokenStream` seen
-        // by the callback. This allows us to avoid producing a `TokenStream`
-        // if it is never needed - for example, a captured `macro_rules!`
-        // argument that is never passed to a proc macro.
-        // In practice token stream creation happens rarely compared to
-        // calls to `collect_tokens` (see some statistics in #78736),
-        // so we are doing as little up-front work as possible.
-        //
-        // This also makes `Parser` very cheap to clone, since
-        // there is no intermediate collection buffer to clone.
-        #[derive(Clone)]
-        struct LazyTokenStreamImpl {
-            start_token: (Token, Spacing),
-            cursor_snapshot: TokenCursor,
-            num_calls: usize,
-            desugar_doc_comments: bool,
-            append_unglued_token: Option<TreeAndSpacing>,
+        let has_outer_attrs = !attrs.attrs.is_empty();
+        let prev_capturing = std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes);
+        let replace_ranges_start = self.capture_state.replace_ranges.len();
+
+        let ret = f(self, attrs.attrs.into());
+
+        self.capture_state.capturing = prev_capturing;
+
+        let (mut ret, trailing) = ret?;
+
+        // When we're not in `capture-cfg` mode, then bail out early if:
+        // 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`)
+        //    so there's nothing for us to do.
+        // 2. Our target already has tokens set (e.g. we've parsed something
+        // like `#[my_attr] $item`. The actual parsing code takes care of prepending
+        // any attributes to the nonterminal, so we don't need to modify the
+        // already captured tokens.
+        // Note that this check is independent of `force_collect`- if we already
+        // have tokens, or can't even store them, then there's never a need to
+        // force collection of new tokens.
+        if !self.capture_cfg && matches!(ret.tokens_mut(), None | Some(Some(_))) {
+            return Ok(ret);
+        }
+
+        // This is very similar to the bail out check at the start of this function.
+        // Now that we've parsed an AST node, we have more information available.
+        if matches!(force_collect, ForceCollect::No)
+            // We now have inner attributes available, so this check is more precise
+            // than `attrs.maybe_needs_tokens()` at the start of the function.
+            // As a result, we don't need to check `R::SUPPORTS_CUSTOM_INNER_ATTRS`
+            && !crate::parser::attr::maybe_needs_tokens(ret.attrs())
+            // Subtle: We call `has_cfg_or_cfg_attr` with the attrs from `ret`.
+            // This ensures that we consider inner attributes (e.g. `#![cfg]`),
+            // which require us to have tokens available
+            // We also call `has_cfg_or_cfg_attr` at the beginning of this function,
+            // but we only bail out if there's no possibility of inner attributes
+            // (!R::SUPPORTS_CUSTOM_INNER_ATTRS)
+            // We only catpure about `#[cfg]` or `#[cfg_attr]` in `capture_cfg`
+            // mode - during normal parsing, we don't need any special capturing
+            // for those attributes, since they're builtin.
+            && !(self.capture_cfg && has_cfg_or_cfg_attr(ret.attrs()))
+        {
+            return Ok(ret);
         }
-        impl CreateTokenStream for LazyTokenStreamImpl {
-            fn create_token_stream(&self) -> TokenStream {
-                // The token produced by the final call to `next` or `next_desugared`
-                // was not actually consumed by the callback. The combination
-                // of chaining the initial token and using `take` produces the desired
-                // result - we produce an empty `TokenStream` if no calls were made,
-                // and omit the final token otherwise.
-                let mut cursor_snapshot = self.cursor_snapshot.clone();
-                let tokens = std::iter::once(self.start_token.clone())
-                    .chain((0..self.num_calls).map(|_| {
-                        if self.desugar_doc_comments {
-                            cursor_snapshot.next_desugared()
-                        } else {
-                            cursor_snapshot.next()
-                        }
-                    }))
-                    .take(self.num_calls);
-
-                make_token_stream(tokens, self.append_unglued_token.clone())
+
+        let mut inner_attr_replace_ranges = Vec::new();
+        // Take the captured ranges for any inner attributes that we parsed.
+        for inner_attr in ret.attrs().iter().filter(|a| a.style == ast::AttrStyle::Inner) {
+            if let Some(attr_range) = self.capture_state.inner_attr_ranges.remove(&inner_attr.id) {
+                inner_attr_replace_ranges.push(attr_range);
+            } else {
+                self.sess
+                    .span_diagnostic
+                    .delay_span_bug(inner_attr.span, "Missing token range for attribute");
             }
         }
 
-        let mut num_calls = self.token_cursor.num_next_calls - cursor_snapshot.num_next_calls;
-        match trailing_token {
+        let replace_ranges_end = self.capture_state.replace_ranges.len();
+
+        let cursor_snapshot_next_calls = cursor_snapshot.num_next_calls;
+        let mut end_pos = self.token_cursor.num_next_calls;
+
+        // Capture a trailing token if requested by the callback 'f'
+        match trailing {
             TrailingToken::None => {}
             TrailingToken::Semi => {
                 assert_eq!(self.token.kind, token::Semi);
-                num_calls += 1;
+                end_pos += 1;
             }
             TrailingToken::MaybeComma => {
                 if self.token.kind == token::Comma {
-                    num_calls += 1;
+                    end_pos += 1;
                 }
             }
         }
 
-        let lazy_impl = LazyTokenStreamImpl {
+        // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens),
+        // then extend the range of captured tokens to include it, since the parser
+        // was not actually bumped past it. When the `LazyTokenStream` gets converted
+        // into a `AttrAnnotatedTokenStream`, we will create the proper token.
+        if self.token_cursor.break_last_token {
+            assert_eq!(
+                trailing,
+                TrailingToken::None,
+                "Cannot set `break_last_token` and have trailing token"
+            );
+            end_pos += 1;
+        }
+
+        let num_calls = end_pos - cursor_snapshot_next_calls;
+
+        // If we have no attributes, then we will never need to
+        // use any replace ranges.
+        let replace_ranges: Box<[ReplaceRange]> = if ret.attrs().is_empty() && !self.capture_cfg {
+            Box::new([])
+        } else {
+            // Grab any replace ranges that occur *inside* the current AST node.
+            // We will perform the actual replacement when we convert the `LazyTokenStream`
+            // to a `AttrAnnotatedTokenStream`
+            let start_calls: u32 = cursor_snapshot_next_calls.try_into().unwrap();
+            self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end]
+                .iter()
+                .cloned()
+                .chain(inner_attr_replace_ranges.clone().into_iter())
+                .map(|(range, tokens)| {
+                    ((range.start - start_calls)..(range.end - start_calls), tokens)
+                })
+                .collect()
+        };
+
+        let tokens = LazyTokenStream::new(LazyTokenStreamImpl {
             start_token,
             num_calls,
             cursor_snapshot,
-            desugar_doc_comments: self.desugar_doc_comments,
-            append_unglued_token: self.token_cursor.append_unglued_token.clone(),
-        };
-        ret.finalize_tokens(LazyTokenStream::new(lazy_impl));
+            break_last_token: self.token_cursor.break_last_token,
+            replace_ranges,
+        });
+
+        // If we support tokens at all
+        if let Some(target_tokens) = ret.tokens_mut() {
+            if let Some(target_tokens) = target_tokens {
+                assert!(
+                    !self.capture_cfg,
+                    "Encountered existing tokens with capture_cfg set: {:?}",
+                    target_tokens
+                );
+            } else {
+                // Store se our newly captured tokens into the AST node
+                *target_tokens = Some(tokens.clone());
+            };
+        }
+
+        let final_attrs = ret.attrs();
+
+        // If `capture_cfg` is set and we're inside a recursive call to
+        // `collect_tokens_trailing_token`, then we need to register a replace range
+        // if we have `#[cfg]` or `#[cfg_attr]`. This allows us to run eager cfg-expansion
+        // on the captured token stream.
+        if self.capture_cfg
+            && matches!(self.capture_state.capturing, Capturing::Yes)
+            && has_cfg_or_cfg_attr(&final_attrs)
+        {
+            let attr_data = AttributesData { attrs: final_attrs.to_vec().into(), tokens };
+
+            // Replace the entire AST node that we just parsed, including attributes,
+            // with a `FlatToken::AttrTarget`. If this AST node is inside an item
+            // that has `#[derive]`, then this will allow us to cfg-expand this
+            // AST node.
+            let start_pos =
+                if has_outer_attrs { attrs.start_pos } else { cursor_snapshot_next_calls };
+            let new_tokens = vec![(FlatToken::AttrTarget(attr_data), Spacing::Alone)];
+
+            assert!(
+                !self.token_cursor.break_last_token,
+                "Should not have unglued last token with cfg attr"
+            );
+            let range: Range<u32> = (start_pos.try_into().unwrap())..(end_pos.try_into().unwrap());
+            self.capture_state.replace_ranges.push((range, new_tokens));
+            self.capture_state.replace_ranges.extend(inner_attr_replace_ranges);
+        }
+
+        // Only clear our `replace_ranges` when we're finished capturing entirely.
+        if matches!(self.capture_state.capturing, Capturing::No) {
+            self.capture_state.replace_ranges.clear();
+            // We don't clear `inner_attr_ranges`, as doing so repeatedly
+            // had a measureable performance impact. Most inner attributes that
+            // we insert will get removed - when we drop the parser, we'll free
+            // up the memory used by any attributes that we didn't remove from the map.
+        }
         Ok(ret)
     }
 }
@@ -143,43 +398,112 @@ impl<'a> Parser<'a> {
 /// Converts a flattened iterator of tokens (including open and close delimiter tokens)
 /// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair
 /// of open and close delims.
+// FIXME(#67062): Currently, we don't parse `None`-delimited groups correctly,
+// which can cause us to end up with mismatched `None` delimiters in our
+// captured tokens. This function contains several hacks to work around this -
+// essentially, we throw away mismatched `None` delimiters when we encounter them.
+// Once we properly parse `None` delimiters, they can be captured just like any
+// other tokens, and these hacks can be removed.
 fn make_token_stream(
-    tokens: impl Iterator<Item = (Token, Spacing)>,
-    append_unglued_token: Option<TreeAndSpacing>,
-) -> TokenStream {
+    mut iter: impl Iterator<Item = (FlatToken, Spacing)>,
+    break_last_token: bool,
+) -> AttrAnnotatedTokenStream {
     #[derive(Debug)]
     struct FrameData {
         open: Span,
-        inner: Vec<(TokenTree, Spacing)>,
+        open_delim: DelimToken,
+        inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
     }
-    let mut stack = vec![FrameData { open: DUMMY_SP, inner: vec![] }];
-    for (token, spacing) in tokens {
+    let mut stack =
+        vec![FrameData { open: DUMMY_SP, open_delim: DelimToken::NoDelim, inner: vec![] }];
+    let mut token_and_spacing = iter.next();
+    while let Some((token, spacing)) = token_and_spacing {
         match token {
-            Token { kind: TokenKind::OpenDelim(_), span } => {
-                stack.push(FrameData { open: span, inner: vec![] });
+            FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
+                stack.push(FrameData { open: span, open_delim: delim, inner: vec![] });
             }
-            Token { kind: TokenKind::CloseDelim(delim), span } => {
-                let frame_data = stack.pop().expect("Token stack was empty!");
+            FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
+                // HACK: If we enconter a mismatched `None` delimiter at the top
+                // level, just ignore it.
+                if matches!(delim, DelimToken::NoDelim)
+                    && (stack.len() == 1
+                        || !matches!(stack.last_mut().unwrap().open_delim, DelimToken::NoDelim))
+                {
+                    token_and_spacing = iter.next();
+                    continue;
+                }
+                let frame_data = stack
+                    .pop()
+                    .unwrap_or_else(|| panic!("Token stack was empty for token: {:?}", token));
+
+                // HACK: If our current frame has a mismatched opening `None` delimiter,
+                // merge our current frame with the one above it. That is, transform
+                // `[ { < first second } third ]` into `[ { first second } third ]`
+                if !matches!(delim, DelimToken::NoDelim)
+                    && matches!(frame_data.open_delim, DelimToken::NoDelim)
+                {
+                    stack.last_mut().unwrap().inner.extend(frame_data.inner);
+                    // Process our closing delimiter again, this time at the previous
+                    // frame in the stack
+                    token_and_spacing = Some((token, spacing));
+                    continue;
+                }
+
+                assert_eq!(
+                    frame_data.open_delim, delim,
+                    "Mismatched open/close delims: open={:?} close={:?}",
+                    frame_data.open, span
+                );
                 let dspan = DelimSpan::from_pair(frame_data.open, span);
-                let stream = TokenStream::new(frame_data.inner);
-                let delimited = TokenTree::Delimited(dspan, delim, stream);
+                let stream = AttrAnnotatedTokenStream::new(frame_data.inner);
+                let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream);
                 stack
                     .last_mut()
-                    .unwrap_or_else(|| panic!("Bottom token frame is missing for tokens!"))
+                    .unwrap_or_else(|| {
+                        panic!("Bottom token frame is missing for token: {:?}", token)
+                    })
                     .inner
                     .push((delimited, Spacing::Alone));
             }
-            token => {
-                stack
-                    .last_mut()
-                    .expect("Bottom token frame is missing!")
-                    .inner
-                    .push((TokenTree::Token(token), spacing));
-            }
+            FlatToken::Token(token) => stack
+                .last_mut()
+                .expect("Bottom token frame is missing!")
+                .inner
+                .push((AttrAnnotatedTokenTree::Token(token), spacing)),
+            FlatToken::AttrTarget(data) => stack
+                .last_mut()
+                .expect("Bottom token frame is missing!")
+                .inner
+                .push((AttrAnnotatedTokenTree::Attributes(data), spacing)),
+            FlatToken::Empty => {}
         }
+        token_and_spacing = iter.next();
+    }
+    // HACK: If we don't have a closing `None` delimiter for our last
+    // frame, merge the frame with the top-level frame. That is,
+    // turn `< first second` into `first second`
+    if stack.len() == 2 && stack[1].open_delim == DelimToken::NoDelim {
+        let temp_buf = stack.pop().unwrap();
+        stack.last_mut().unwrap().inner.extend(temp_buf.inner);
     }
     let mut final_buf = stack.pop().expect("Missing final buf!");
-    final_buf.inner.extend(append_unglued_token);
+    if break_last_token {
+        let (last_token, spacing) = final_buf.inner.pop().unwrap();
+        if let AttrAnnotatedTokenTree::Token(last_token) = last_token {
+            let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
+
+            // A 'unglued' token is always two ASCII characters
+            let mut first_span = last_token.span.shrink_to_lo();
+            first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
+
+            final_buf.inner.push((
+                AttrAnnotatedTokenTree::Token(Token::new(unglued_first, first_span)),
+                spacing,
+            ));
+        } else {
+            panic!("Unexpected last token {:?}", last_token)
+        }
+    }
     assert!(stack.is_empty(), "Stack should be empty: final_buf={:?} stack={:?}", final_buf, stack);
-    TokenStream::new(final_buf.inner)
+    AttrAnnotatedTokenStream::new(final_buf.inner)
 }
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index f4ab3260d1a..70a5ac6f15e 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -640,7 +640,7 @@ impl<'a> Parser<'a> {
                     }
                 }
                 Err(mut err) => {
-                    // We could't parse generic parameters, unlikely to be a turbofish. Rely on
+                    // We couldn't parse generic parameters, unlikely to be a turbofish. Rely on
                     // generic parse error instead.
                     err.cancel();
                     *self = snapshot;
@@ -666,21 +666,23 @@ impl<'a> Parser<'a> {
                     );
                     match x {
                         Ok((_, _, false)) => {
-                            self.bump(); // `>`
-                            match self.parse_expr() {
-                                Ok(_) => {
-                                    e.span_suggestion_verbose(
-                                        binop.span.shrink_to_lo(),
-                                        TURBOFISH_SUGGESTION_STR,
-                                        "::".to_string(),
-                                        Applicability::MaybeIncorrect,
-                                    );
-                                    e.emit();
-                                    *expr = self.mk_expr_err(expr.span.to(self.prev_token.span));
-                                    return Ok(());
-                                }
-                                Err(mut err) => {
-                                    err.cancel();
+                            if self.eat(&token::Gt) {
+                                match self.parse_expr() {
+                                    Ok(_) => {
+                                        e.span_suggestion_verbose(
+                                            binop.span.shrink_to_lo(),
+                                            TURBOFISH_SUGGESTION_STR,
+                                            "::".to_string(),
+                                            Applicability::MaybeIncorrect,
+                                        );
+                                        e.emit();
+                                        *expr =
+                                            self.mk_expr_err(expr.span.to(self.prev_token.span));
+                                        return Ok(());
+                                    }
+                                    Err(mut err) => {
+                                        err.cancel();
+                                    }
                                 }
                             }
                         }
@@ -1242,7 +1244,7 @@ impl<'a> Parser<'a> {
         let is_question = self.eat(&token::Question); // Handle `await? <expr>`.
         let expr = if self.token == token::OpenDelim(token::Brace) {
             // Handle `await { <expr> }`.
-            // This needs to be handled separatedly from the next arm to avoid
+            // This needs to be handled separately from the next arm to avoid
             // interpreting `await { <expr> }?` as `<expr>?.await`.
             self.parse_block_expr(None, self.token.span, BlockCheckMode::Default, AttrVec::new())
         } else {
@@ -1613,42 +1615,82 @@ impl<'a> Parser<'a> {
                 Applicability::HasPlaceholders,
             );
             return Some(ident);
-        } else if let PatKind::Ident(_, ident, _) = pat.kind {
-            if require_name
-                && (self.token == token::Comma
-                    || self.token == token::Lt
-                    || self.token == token::CloseDelim(token::Paren))
-            {
-                // `fn foo(a, b) {}`, `fn foo(a<x>, b<y>) {}` or `fn foo(usize, usize) {}`
-                if first_param {
-                    err.span_suggestion(
-                        pat.span,
-                        "if this is a `self` type, give it a parameter name",
-                        format!("self: {}", ident),
-                        Applicability::MaybeIncorrect,
-                    );
+        } else if require_name
+            && (self.token == token::Comma
+                || self.token == token::Lt
+                || self.token == token::CloseDelim(token::Paren))
+        {
+            let rfc_note = "anonymous parameters are removed in the 2018 edition (see RFC 1685)";
+
+            let (ident, self_sugg, param_sugg, type_sugg) = match pat.kind {
+                PatKind::Ident(_, ident, _) => (
+                    ident,
+                    format!("self: {}", ident),
+                    format!("{}: TypeName", ident),
+                    format!("_: {}", ident),
+                ),
+                // Also catches `fn foo(&a)`.
+                PatKind::Ref(ref pat, mutab)
+                    if matches!(pat.clone().into_inner().kind, PatKind::Ident(..)) =>
+                {
+                    match pat.clone().into_inner().kind {
+                        PatKind::Ident(_, ident, _) => {
+                            let mutab = mutab.prefix_str();
+                            (
+                                ident,
+                                format!("self: &{}{}", mutab, ident),
+                                format!("{}: &{}TypeName", ident, mutab),
+                                format!("_: &{}{}", mutab, ident),
+                            )
+                        }
+                        _ => unreachable!(),
+                    }
                 }
-                // Avoid suggesting that `fn foo(HashMap<u32>)` is fixed with a change to
-                // `fn foo(HashMap: TypeName<u32>)`.
-                if self.token != token::Lt {
-                    err.span_suggestion(
-                        pat.span,
-                        "if this is a parameter name, give it a type",
-                        format!("{}: TypeName", ident),
-                        Applicability::HasPlaceholders,
-                    );
+                _ => {
+                    // Otherwise, try to get a type and emit a suggestion.
+                    if let Some(ty) = pat.to_ty() {
+                        err.span_suggestion_verbose(
+                            pat.span,
+                            "explicitly ignore the parameter name",
+                            format!("_: {}", pprust::ty_to_string(&ty)),
+                            Applicability::MachineApplicable,
+                        );
+                        err.note(rfc_note);
+                    }
+
+                    return None;
                 }
+            };
+
+            // `fn foo(a, b) {}`, `fn foo(a<x>, b<y>) {}` or `fn foo(usize, usize) {}`
+            if first_param {
                 err.span_suggestion(
                     pat.span,
-                    "if this is a type, explicitly ignore the parameter name",
-                    format!("_: {}", ident),
-                    Applicability::MachineApplicable,
+                    "if this is a `self` type, give it a parameter name",
+                    self_sugg,
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            // Avoid suggesting that `fn foo(HashMap<u32>)` is fixed with a change to
+            // `fn foo(HashMap: TypeName<u32>)`.
+            if self.token != token::Lt {
+                err.span_suggestion(
+                    pat.span,
+                    "if this is a parameter name, give it a type",
+                    param_sugg,
+                    Applicability::HasPlaceholders,
                 );
-                err.note("anonymous parameters are removed in the 2018 edition (see RFC 1685)");
-
-                // Don't attempt to recover by using the `X` in `X<Y>` as the parameter name.
-                return if self.token == token::Lt { None } else { Some(ident) };
             }
+            err.span_suggestion(
+                pat.span,
+                "if this is a type, explicitly ignore the parameter name",
+                type_sugg,
+                Applicability::MachineApplicable,
+            );
+            err.note(rfc_note);
+
+            // Don't attempt to recover by using the `X` in `X<Y>` as the parameter name.
+            return if self.token == token::Lt { None } else { Some(ident) };
         }
         None
     }
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 28bfaea4555..e155b3fa773 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1,4 +1,4 @@
-use super::pat::{GateOr, RecoverComma, PARAM_EXPECTED};
+use super::pat::{RecoverComma, PARAM_EXPECTED};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{AttrWrapper, BlockMode, ForceCollect, Parser, PathStyle, Restrictions, TokenType};
 use super::{SemiColonMode, SeqSep, TokenExpectType, TrailingToken};
@@ -10,7 +10,7 @@ use rustc_ast::tokenstream::Spacing;
 use rustc_ast::util::classify;
 use rustc_ast::util::literal::LitError;
 use rustc_ast::util::parser::{prec_let_scrutinee_needs_par, AssocOp, Fixity};
-use rustc_ast::{self as ast, AttrStyle, AttrVec, CaptureBy, Field, Lit, UnOp, DUMMY_NODE_ID};
+use rustc_ast::{self as ast, AttrStyle, AttrVec, CaptureBy, ExprField, Lit, UnOp, DUMMY_NODE_ID};
 use rustc_ast::{AnonConst, BinOp, BinOpKind, FnDecl, FnRetTy, MacCall, Param, Ty, TyKind};
 use rustc_ast::{Arm, Async, BlockCheckMode, Expr, ExprKind, Label, Movability, RangeLimits};
 use rustc_ast_pretty::pprust;
@@ -92,7 +92,22 @@ impl<'a> Parser<'a> {
         self.parse_expr_res(Restrictions::empty(), None)
     }
 
-    pub(super) fn parse_anon_const_expr(&mut self) -> PResult<'a, AnonConst> {
+    /// Parses an expression, forcing tokens to be collected
+    pub fn parse_expr_force_collect(&mut self) -> PResult<'a, P<Expr>> {
+        // If we have outer attributes, then the call to `collect_tokens_trailing_token`
+        // will be made for us.
+        if matches!(self.token.kind, TokenKind::Pound | TokenKind::DocComment(..)) {
+            self.parse_expr()
+        } else {
+            // If we don't have outer attributes, then we need to ensure
+            // that collection happens by using `collect_tokens_no_attrs`.
+            // Expression don't support custom inner attributes, so `parse_expr`
+            // will never try to collect tokens if we don't have outer attributes.
+            self.collect_tokens_no_attrs(|this| this.parse_expr())
+        }
+    }
+
+    pub fn parse_anon_const_expr(&mut self) -> PResult<'a, AnonConst> {
         self.parse_expr().map(|value| AnonConst { id: DUMMY_NODE_ID, value })
     }
 
@@ -1803,7 +1818,7 @@ impl<'a> Parser<'a> {
     /// The `let` token has already been eaten.
     fn parse_let_expr(&mut self, attrs: AttrVec) -> PResult<'a, P<Expr>> {
         let lo = self.prev_token.span;
-        let pat = self.parse_pat_allow_top_alt(None, GateOr::No, RecoverComma::Yes)?;
+        let pat = self.parse_pat_allow_top_alt(None, RecoverComma::Yes)?;
         self.expect(&token::Eq)?;
         let expr = self.with_res(self.restrictions | Restrictions::NO_STRUCT_LITERAL, |this| {
             this.parse_assoc_expr_with(1 + prec_let_scrutinee_needs_par(), None.into())
@@ -1866,7 +1881,7 @@ impl<'a> Parser<'a> {
             _ => None,
         };
 
-        let pat = self.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::Yes)?;
+        let pat = self.parse_pat_allow_top_alt(None, RecoverComma::Yes)?;
         if !self.eat_keyword(kw::In) {
             self.error_missing_in_for_loop();
         }
@@ -1973,11 +1988,107 @@ impl<'a> Parser<'a> {
         Ok(self.mk_expr(lo.to(hi), ExprKind::Match(scrutinee, arms), attrs))
     }
 
+    /// Attempt to recover from match arm body with statements and no surrounding braces.
+    fn parse_arm_body_missing_braces(
+        &mut self,
+        first_expr: &P<Expr>,
+        arrow_span: Span,
+    ) -> Option<P<Expr>> {
+        if self.token.kind != token::Semi {
+            return None;
+        }
+        let start_snapshot = self.clone();
+        let semi_sp = self.token.span;
+        self.bump(); // `;`
+        let mut stmts =
+            vec![self.mk_stmt(first_expr.span, ast::StmtKind::Expr(first_expr.clone()))];
+        let err = |this: &mut Parser<'_>, stmts: Vec<ast::Stmt>| {
+            let span = stmts[0].span.to(stmts[stmts.len() - 1].span);
+            let mut err = this.struct_span_err(span, "`match` arm body without braces");
+            let (these, s, are) =
+                if stmts.len() > 1 { ("these", "s", "are") } else { ("this", "", "is") };
+            err.span_label(
+                span,
+                &format!(
+                    "{these} statement{s} {are} not surrounded by a body",
+                    these = these,
+                    s = s,
+                    are = are
+                ),
+            );
+            err.span_label(arrow_span, "while parsing the `match` arm starting here");
+            if stmts.len() > 1 {
+                err.multipart_suggestion(
+                    &format!("surround the statement{} with a body", s),
+                    vec![
+                        (span.shrink_to_lo(), "{ ".to_string()),
+                        (span.shrink_to_hi(), " }".to_string()),
+                    ],
+                    Applicability::MachineApplicable,
+                );
+            } else {
+                err.span_suggestion(
+                    semi_sp,
+                    "use a comma to end a `match` arm expression",
+                    ",".to_string(),
+                    Applicability::MachineApplicable,
+                );
+            }
+            err.emit();
+            this.mk_expr_err(span)
+        };
+        // We might have either a `,` -> `;` typo, or a block without braces. We need
+        // a more subtle parsing strategy.
+        loop {
+            if self.token.kind == token::CloseDelim(token::Brace) {
+                // We have reached the closing brace of the `match` expression.
+                return Some(err(self, stmts));
+            }
+            if self.token.kind == token::Comma {
+                *self = start_snapshot;
+                return None;
+            }
+            let pre_pat_snapshot = self.clone();
+            match self.parse_pat_no_top_alt(None) {
+                Ok(_pat) => {
+                    if self.token.kind == token::FatArrow {
+                        // Reached arm end.
+                        *self = pre_pat_snapshot;
+                        return Some(err(self, stmts));
+                    }
+                }
+                Err(mut err) => {
+                    err.cancel();
+                }
+            }
+
+            *self = pre_pat_snapshot;
+            match self.parse_stmt_without_recovery(true, ForceCollect::No) {
+                // Consume statements for as long as possible.
+                Ok(Some(stmt)) => {
+                    stmts.push(stmt);
+                }
+                Ok(None) => {
+                    *self = start_snapshot;
+                    break;
+                }
+                // We couldn't parse either yet another statement missing it's
+                // enclosing block nor the next arm's pattern or closing brace.
+                Err(mut stmt_err) => {
+                    stmt_err.cancel();
+                    *self = start_snapshot;
+                    break;
+                }
+            }
+        }
+        None
+    }
+
     pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> {
         let attrs = self.parse_outer_attributes()?;
         self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
             let lo = this.token.span;
-            let pat = this.parse_pat_allow_top_alt(None, GateOr::No, RecoverComma::Yes)?;
+            let pat = this.parse_pat_allow_top_alt(None, RecoverComma::Yes)?;
             let guard = if this.eat_keyword(kw::If) {
                 let if_span = this.prev_token.span;
                 let cond = this.parse_expr()?;
@@ -2007,6 +2118,21 @@ impl<'a> Parser<'a> {
 
             if require_comma {
                 let sm = this.sess.source_map();
+                if let Some(body) = this.parse_arm_body_missing_braces(&expr, arrow_span) {
+                    let span = body.span;
+                    return Ok((
+                        ast::Arm {
+                            attrs,
+                            pat,
+                            guard,
+                            body,
+                            span,
+                            id: DUMMY_NODE_ID,
+                            is_placeholder: false,
+                        },
+                        TrailingToken::None,
+                    ));
+                }
                 this.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Brace)]).map_err(
                     |mut err| {
                         match (sm.span_to_lines(expr.span), sm.span_to_lines(arm_start_span)) {
@@ -2205,7 +2331,7 @@ impl<'a> Parser<'a> {
             }
 
             let recovery_field = self.find_struct_error_after_field_looking_code();
-            let parsed_field = match self.parse_field() {
+            let parsed_field = match self.parse_expr_field() {
                 Ok(f) => Some(f),
                 Err(mut e) => {
                     if pth == kw::Async {
@@ -2262,18 +2388,22 @@ impl<'a> Parser<'a> {
 
         let span = pth.span.to(self.token.span);
         self.expect(&token::CloseDelim(token::Brace))?;
-        let expr = if recover_async { ExprKind::Err } else { ExprKind::Struct(pth, fields, base) };
+        let expr = if recover_async {
+            ExprKind::Err
+        } else {
+            ExprKind::Struct(P(ast::StructExpr { path: pth, fields, rest: base }))
+        };
         Ok(self.mk_expr(span, expr, attrs))
     }
 
     /// Use in case of error after field-looking code: `S { foo: () with a }`.
-    fn find_struct_error_after_field_looking_code(&self) -> Option<Field> {
+    fn find_struct_error_after_field_looking_code(&self) -> Option<ExprField> {
         match self.token.ident() {
             Some((ident, is_raw))
                 if (is_raw || !ident.is_reserved())
                     && self.look_ahead(1, |t| *t == token::Colon) =>
             {
-                Some(ast::Field {
+                Some(ast::ExprField {
                     ident,
                     span: self.token.span,
                     expr: self.mk_expr_err(self.token.span),
@@ -2307,7 +2437,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses `ident (COLON expr)?`.
-    fn parse_field(&mut self) -> PResult<'a, Field> {
+    fn parse_expr_field(&mut self) -> PResult<'a, ExprField> {
         let attrs = self.parse_outer_attributes()?;
         self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
             let lo = this.token.span;
@@ -2327,7 +2457,7 @@ impl<'a> Parser<'a> {
             };
 
             Ok((
-                ast::Field {
+                ast::ExprField {
                     ident,
                     span: lo.to(expr.span),
                     expr,
@@ -2451,19 +2581,17 @@ impl<'a> Parser<'a> {
         attrs: AttrWrapper,
         f: impl FnOnce(&mut Self, Vec<ast::Attribute>) -> PResult<'a, P<Expr>>,
     ) -> PResult<'a, P<Expr>> {
-        // FIXME - come up with a nice way to properly forward `ForceCollect`from
-        // the nonterminal parsing code. TThis approach iscorrect, but will cause
-        // us to unnecessarily capture tokens for exprs that have only builtin
-        // attributes. Revisit this before #![feature(stmt_expr_attributes)] is stabilized
-        let force_collect = if attrs.is_empty() { ForceCollect::No } else { ForceCollect::Yes };
-        self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
+        self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
             let res = f(this, attrs)?;
             let trailing = if this.restrictions.contains(Restrictions::STMT_EXPR)
                 && this.token.kind == token::Semi
             {
                 TrailingToken::Semi
             } else {
-                TrailingToken::None
+                // FIXME - pass this through from the place where we know
+                // we need a comma, rather than assuming that `#[attr] expr,`
+                // always captures a trailing comma
+                TrailingToken::MaybeComma
             };
             Ok((res, trailing))
         })
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 9668a24bf8a..299b9a959c5 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -9,7 +9,7 @@ use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID};
 use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind};
 use rustc_ast::{BindingMode, Block, FnDecl, FnSig, Param, SelfKind};
-use rustc_ast::{EnumDef, Generics, StructField, TraitRef, Ty, TyKind, Variant, VariantData};
+use rustc_ast::{EnumDef, FieldDef, Generics, TraitRef, Ty, TyKind, Variant, VariantData};
 use rustc_ast::{FnHeader, ForeignItem, Path, PathSegment, Visibility, VisibilityKind};
 use rustc_ast::{MacArgs, MacCall, MacDelimiter};
 use rustc_ast_pretty::pprust;
@@ -103,20 +103,11 @@ impl<'a> Parser<'a> {
         // over when we bump the parser
         if let token::Interpolated(nt) = &self.token.kind {
             if let token::NtItem(item) = &**nt {
-                let item = item.clone();
-
-                return self.collect_tokens_trailing_token(
-                    attrs,
-                    force_collect,
-                    |this, mut attrs| {
-                        let mut item = item;
-                        mem::swap(&mut item.attrs, &mut attrs);
-                        item.attrs.extend(attrs);
-                        // Bump the parser so the we capture the token::Interpolated
-                        this.bump();
-                        Ok((Some(item.into_inner()), TrailingToken::None))
-                    },
-                );
+                let mut item = item.clone();
+                self.bump();
+
+                attrs.prepend_to_nt_inner(&mut item.attrs);
+                return Ok(Some(item.into_inner()));
             }
         };
 
@@ -204,6 +195,7 @@ impl<'a> Parser<'a> {
         def: &mut Defaultness,
         req_name: ReqName,
     ) -> PResult<'a, Option<ItemInfo>> {
+        let def_final = def == &Defaultness::Final;
         let mut def = || mem::replace(def, Defaultness::Final);
 
         let info = if self.eat_keyword(kw::Use) {
@@ -226,7 +218,7 @@ impl<'a> Parser<'a> {
             }
 
             (Ident::invalid(), ItemKind::Use(tree))
-        } else if self.check_fn_front_matter() {
+        } else if self.check_fn_front_matter(def_final) {
             // FUNCTION ITEM
             let (ident, sig, generics, body) = self.parse_fn(attrs, req_name, lo)?;
             (ident, ItemKind::Fn(box FnKind(def(), sig, generics, body)))
@@ -529,7 +521,7 @@ impl<'a> Parser<'a> {
 
         generics.where_clause = self.parse_where_clause()?;
 
-        let impl_items = self.parse_item_list(attrs, |p| p.parse_impl_item())?;
+        let impl_items = self.parse_item_list(attrs, |p| p.parse_impl_item(ForceCollect::No))?;
 
         let item_kind = match ty_second {
             Some(ty_second) => {
@@ -717,22 +709,32 @@ impl<'a> Parser<'a> {
         } else {
             // It's a normal trait.
             tps.where_clause = self.parse_where_clause()?;
-            let items = self.parse_item_list(attrs, |p| p.parse_trait_item())?;
+            let items = self.parse_item_list(attrs, |p| p.parse_trait_item(ForceCollect::No))?;
             Ok((ident, ItemKind::Trait(box TraitKind(is_auto, unsafety, tps, bounds, items))))
         }
     }
 
-    pub fn parse_impl_item(&mut self) -> PResult<'a, Option<Option<P<AssocItem>>>> {
-        self.parse_assoc_item(|_| true)
+    pub fn parse_impl_item(
+        &mut self,
+        force_collect: ForceCollect,
+    ) -> PResult<'a, Option<Option<P<AssocItem>>>> {
+        self.parse_assoc_item(|_| true, force_collect)
     }
 
-    pub fn parse_trait_item(&mut self) -> PResult<'a, Option<Option<P<AssocItem>>>> {
-        self.parse_assoc_item(|edition| edition >= Edition::Edition2018)
+    pub fn parse_trait_item(
+        &mut self,
+        force_collect: ForceCollect,
+    ) -> PResult<'a, Option<Option<P<AssocItem>>>> {
+        self.parse_assoc_item(|edition| edition >= Edition::Edition2018, force_collect)
     }
 
     /// Parses associated items.
-    fn parse_assoc_item(&mut self, req_name: ReqName) -> PResult<'a, Option<Option<P<AssocItem>>>> {
-        Ok(self.parse_item_(req_name, ForceCollect::No)?.map(
+    fn parse_assoc_item(
+        &mut self,
+        req_name: ReqName,
+        force_collect: ForceCollect,
+    ) -> PResult<'a, Option<Option<P<AssocItem>>>> {
+        Ok(self.parse_item_(req_name, force_collect)?.map(
             |Item { attrs, id, span, vis, ident, kind, tokens }| {
                 let kind = match AssocItemKind::try_from(kind) {
                     Ok(kind) => kind,
@@ -917,14 +919,17 @@ impl<'a> Parser<'a> {
         unsafety: Unsafe,
     ) -> PResult<'a, ItemInfo> {
         let abi = self.parse_abi(); // ABI?
-        let items = self.parse_item_list(attrs, |p| p.parse_foreign_item())?;
+        let items = self.parse_item_list(attrs, |p| p.parse_foreign_item(ForceCollect::No))?;
         let module = ast::ForeignMod { unsafety, abi, items };
         Ok((Ident::invalid(), ItemKind::ForeignMod(module)))
     }
 
     /// Parses a foreign item (one in an `extern { ... }` block).
-    pub fn parse_foreign_item(&mut self) -> PResult<'a, Option<Option<P<ForeignItem>>>> {
-        Ok(self.parse_item_(|_| true, ForceCollect::No)?.map(
+    pub fn parse_foreign_item(
+        &mut self,
+        force_collect: ForceCollect,
+    ) -> PResult<'a, Option<Option<P<ForeignItem>>>> {
+        Ok(self.parse_item_(|_| true, force_collect)?.map(
             |Item { attrs, id, span, vis, ident, kind, tokens }| {
                 let kind = match ForeignItemKind::try_from(kind) {
                     Ok(kind) => kind,
@@ -1231,14 +1236,12 @@ impl<'a> Parser<'a> {
         Ok((class_name, ItemKind::Union(vdata, generics)))
     }
 
-    fn parse_record_struct_body(
-        &mut self,
-    ) -> PResult<'a, (Vec<StructField>, /* recovered */ bool)> {
+    fn parse_record_struct_body(&mut self) -> PResult<'a, (Vec<FieldDef>, /* recovered */ bool)> {
         let mut fields = Vec::new();
         let mut recovered = false;
         if self.eat(&token::OpenDelim(token::Brace)) {
             while self.token != token::CloseDelim(token::Brace) {
-                let field = self.parse_struct_decl_field().map_err(|e| {
+                let field = self.parse_field_def().map_err(|e| {
                     self.consume_block(token::Brace, ConsumeClosingDelim::No);
                     recovered = true;
                     e
@@ -1263,7 +1266,7 @@ impl<'a> Parser<'a> {
         Ok((fields, recovered))
     }
 
-    fn parse_tuple_struct_body(&mut self) -> PResult<'a, Vec<StructField>> {
+    fn parse_tuple_struct_body(&mut self) -> PResult<'a, Vec<FieldDef>> {
         // This is the case where we find `struct Foo<T>(T) where T: Copy;`
         // Unit like structs are handled in parse_item_struct function
         self.parse_paren_comma_seq(|p| {
@@ -1274,7 +1277,7 @@ impl<'a> Parser<'a> {
                 let ty = p.parse_ty()?;
 
                 Ok((
-                    StructField {
+                    FieldDef {
                         span: lo.to(ty.span),
                         vis,
                         ident: None,
@@ -1291,7 +1294,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses an element of a struct declaration.
-    fn parse_struct_decl_field(&mut self) -> PResult<'a, StructField> {
+    fn parse_field_def(&mut self) -> PResult<'a, FieldDef> {
         let attrs = self.parse_outer_attributes()?;
         self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
             let lo = this.token.span;
@@ -1306,7 +1309,7 @@ impl<'a> Parser<'a> {
         lo: Span,
         vis: Visibility,
         attrs: Vec<Attribute>,
-    ) -> PResult<'a, StructField> {
+    ) -> PResult<'a, FieldDef> {
         let mut seen_comma: bool = false;
         let a_var = self.parse_name_and_ty(lo, vis, attrs)?;
         if self.token == token::Comma {
@@ -1398,11 +1401,11 @@ impl<'a> Parser<'a> {
         lo: Span,
         vis: Visibility,
         attrs: Vec<Attribute>,
-    ) -> PResult<'a, StructField> {
+    ) -> PResult<'a, FieldDef> {
         let name = self.parse_ident_common(false)?;
         self.expect(&token::Colon)?;
         let ty = self.parse_ty()?;
-        Ok(StructField {
+        Ok(FieldDef {
             span: lo.to(self.prev_token.span),
             ident: Some(name),
             vis,
@@ -1445,7 +1448,7 @@ impl<'a> Parser<'a> {
         Ok((ident, ItemKind::MacroDef(ast::MacroDef { body, macro_rules: false })))
     }
 
-    /// Is this unambiguously the start of a `macro_rules! foo` item defnition?
+    /// Is this unambiguously the start of a `macro_rules! foo` item definition?
     fn is_macro_rules_item(&mut self) -> bool {
         self.check_keyword(kw::MacroRules)
             && self.look_ahead(1, |t| *t == token::Not)
@@ -1636,18 +1639,27 @@ impl<'a> Parser<'a> {
     }
 
     /// Is the current token the start of an `FnHeader` / not a valid parse?
-    pub(super) fn check_fn_front_matter(&mut self) -> bool {
+    ///
+    /// `check_pub` adds additional `pub` to the checks in case users place it
+    /// wrongly, can be used to ensure `pub` never comes after `default`.
+    pub(super) fn check_fn_front_matter(&mut self, check_pub: bool) -> bool {
         // We use an over-approximation here.
         // `const const`, `fn const` won't parse, but we're not stepping over other syntax either.
-        const QUALS: [Symbol; 4] = [kw::Const, kw::Async, kw::Unsafe, kw::Extern];
+        // `pub` is added in case users got confused with the ordering like `async pub fn`,
+        // only if it wasn't preceeded by `default` as `default pub` is invalid.
+        let quals: &[Symbol] = if check_pub {
+            &[kw::Pub, kw::Const, kw::Async, kw::Unsafe, kw::Extern]
+        } else {
+            &[kw::Const, kw::Async, kw::Unsafe, kw::Extern]
+        };
         self.check_keyword(kw::Fn) // Definitely an `fn`.
             // `$qual fn` or `$qual $qual`:
-            || QUALS.iter().any(|&kw| self.check_keyword(kw))
+            || quals.iter().any(|&kw| self.check_keyword(kw))
                 && self.look_ahead(1, |t| {
                     // `$qual fn`, e.g. `const fn` or `async fn`.
                     t.is_keyword(kw::Fn)
                     // Two qualifiers `$qual $qual` is enough, e.g. `async unsafe`.
-                    || t.is_non_raw_ident_where(|i| QUALS.contains(&i.name)
+                    || t.is_non_raw_ident_where(|i| quals.contains(&i.name)
                         // Rule out 2015 `const async: T = val`.
                         && i.is_reserved()
                         // Rule out unsafe extern block.
@@ -1668,6 +1680,7 @@ impl<'a> Parser<'a> {
     /// FnFrontMatter = FnQual "fn" ;
     /// ```
     pub(super) fn parse_fn_front_matter(&mut self) -> PResult<'a, FnHeader> {
+        let sp_start = self.token.span;
         let constness = self.parse_constness();
         let asyncness = self.parse_asyncness();
         let unsafety = self.parse_unsafety();
@@ -1681,8 +1694,27 @@ impl<'a> Parser<'a> {
             // It is possible for `expect_one_of` to recover given the contents of
             // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't
             // account for this.
-            if !self.expect_one_of(&[], &[])? {
-                unreachable!()
+            match self.expect_one_of(&[], &[]) {
+                Ok(true) => {}
+                Ok(false) => unreachable!(),
+                Err(mut err) => {
+                    // Recover incorrect visibility order such as `async pub`.
+                    if self.check_keyword(kw::Pub) {
+                        let sp = sp_start.to(self.prev_token.span);
+                        if let Ok(snippet) = self.span_to_snippet(sp) {
+                            let vis = self.parse_visibility(FollowedByType::No)?;
+                            let vs = pprust::vis_to_string(&vis);
+                            let vs = vs.trim_end();
+                            err.span_suggestion(
+                                sp_start.to(self.prev_token.span),
+                                &format!("visibility `{}` must come before `{}`", vs, snippet),
+                                format!("{} {}", vs, snippet),
+                                Applicability::MachineApplicable,
+                            );
+                        }
+                    }
+                    return Err(err);
+                }
             }
         }
 
@@ -1757,8 +1789,9 @@ impl<'a> Parser<'a> {
             let (pat, ty) = if is_name_required || this.is_named_param() {
                 debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required);
 
-                let pat = this.parse_fn_param_pat()?;
-                if let Err(mut err) = this.expect(&token::Colon) {
+                let (pat, colon) = this.parse_fn_param_pat_colon()?;
+                if !colon {
+                    let mut err = this.unexpected::<()>().unwrap_err();
                     return if let Some(ident) =
                         this.parameter_without_type(&mut err, pat, is_name_required, first_param)
                     {
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 4cc2224d27e..ed95a5661b1 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -14,18 +14,21 @@ use crate::lexer::UnmatchedBrace;
 pub use attr_wrapper::AttrWrapper;
 pub use diagnostics::AttemptLocalParseRecovery;
 use diagnostics::Error;
-pub use pat::{GateOr, RecoverComma};
+pub use pat::RecoverComma;
 pub use path::PathStyle;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, DelimToken, Token, TokenKind};
+use rustc_ast::tokenstream::AttributesData;
 use rustc_ast::tokenstream::{self, DelimSpan, Spacing};
-use rustc_ast::tokenstream::{TokenStream, TokenTree, TreeAndSpacing};
+use rustc_ast::tokenstream::{TokenStream, TokenTree};
+use rustc_ast::AttrId;
 use rustc_ast::DUMMY_NODE_ID;
 use rustc_ast::{self as ast, AnonConst, AstLike, AttrStyle, AttrVec, Const, CrateSugar, Extern};
 use rustc_ast::{Async, Expr, ExprKind, MacArgs, MacDelimiter, Mutability, StrLit, Unsafe};
 use rustc_ast::{Visibility, VisibilityKind};
 use rustc_ast_pretty::pprust;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::PResult;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError};
@@ -34,6 +37,7 @@ use rustc_span::source_map::{Span, DUMMY_SP};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use tracing::debug;
 
+use std::ops::Range;
 use std::{cmp, mem, slice};
 
 bitflags::bitflags! {
@@ -64,6 +68,7 @@ pub enum ForceCollect {
     No,
 }
 
+#[derive(Debug, Eq, PartialEq)]
 pub enum TrailingToken {
     None,
     Semi,
@@ -111,6 +116,7 @@ pub struct Parser<'a> {
     pub token_spacing: Spacing,
     /// The previous token.
     pub prev_token: Token,
+    pub capture_cfg: bool,
     restrictions: Restrictions,
     expected_tokens: Vec<TokenType>,
     // Important: This must only be advanced from `next_tok`
@@ -134,6 +140,44 @@ pub struct Parser<'a> {
     pub last_type_ascription: Option<(Span, bool /* likely path typo */)>,
     /// If present, this `Parser` is not parsing Rust code but rather a macro call.
     subparser_name: Option<&'static str>,
+    capture_state: CaptureState,
+}
+
+/// Indicates a range of tokens that should be replaced by
+/// the tokens in the provided vector. This is used in two
+/// places during token collection:
+///
+/// 1. During the parsing of an AST node that may have a `#[derive]`
+/// attribute, we parse a nested AST node that has `#[cfg]` or `#[cfg_attr]`
+/// In this case, we use a `ReplaceRange` to replace the entire inner AST node
+/// with `FlatToken::AttrTarget`, allowing us to perform eager cfg-expansion
+/// on a `AttrAnnotatedTokenStream`
+///
+/// 2. When we parse an inner attribute while collecting tokens. We
+/// remove inner attributes from the token stream entirely, and
+/// instead track them through the `attrs` field on the AST node.
+/// This allows us to easily manipulate them (for example, removing
+/// the first macro inner attribute to invoke a proc-macro).
+/// When create a `TokenStream`, the inner attributes get inserted
+/// into the proper place in the token stream.
+pub type ReplaceRange = (Range<u32>, Vec<(FlatToken, Spacing)>);
+
+/// Controls how we capture tokens. Capturing can be expensive,
+/// so we try to avoid performing capturing in cases where
+/// we will never need a `AttrAnnotatedTokenStream`
+#[derive(Copy, Clone)]
+pub enum Capturing {
+    /// We aren't performing any capturing - this is the default mode.
+    No,
+    /// We are capturing tokens
+    Yes,
+}
+
+#[derive(Clone)]
+struct CaptureState {
+    capturing: Capturing,
+    replace_ranges: Vec<ReplaceRange>,
+    inner_attr_ranges: FxHashMap<AttrId, ReplaceRange>,
 }
 
 impl<'a> Drop for Parser<'a> {
@@ -167,11 +211,11 @@ struct TokenCursor {
     // want to capture just the first 'unglued' token.
     // For example, capturing the `Vec<u8>`
     // in `Option<Vec<u8>>` requires us to unglue
-    // the trailing `>>` token. The `append_unglued_token`
+    // the trailing `>>` token. The `break_last_token`
     // field is used to track this token - it gets
     // appended to the captured stream when
     // we evaluate a `LazyTokenStream`
-    append_unglued_token: Option<TreeAndSpacing>,
+    break_last_token: bool,
 }
 
 #[derive(Clone)]
@@ -188,9 +232,9 @@ impl TokenCursorFrame {
         TokenCursorFrame {
             delim,
             span,
-            open_delim: delim == token::NoDelim,
+            open_delim: false,
             tree_cursor: tts.into_trees(),
-            close_delim: delim == token::NoDelim,
+            close_delim: false,
         }
     }
 }
@@ -364,19 +408,24 @@ impl<'a> Parser<'a> {
         desugar_doc_comments: bool,
         subparser_name: Option<&'static str>,
     ) -> Self {
+        let mut start_frame = TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens);
+        start_frame.open_delim = true;
+        start_frame.close_delim = true;
+
         let mut parser = Parser {
             sess,
             token: Token::dummy(),
             token_spacing: Spacing::Alone,
             prev_token: Token::dummy(),
+            capture_cfg: false,
             restrictions: Restrictions::empty(),
             expected_tokens: Vec::new(),
             token_cursor: TokenCursor {
-                frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens),
+                frame: start_frame,
                 stack: Vec::new(),
                 num_next_calls: 0,
                 desugar_doc_comments,
-                append_unglued_token: None,
+                break_last_token: false,
             },
             desugar_doc_comments,
             unmatched_angle_bracket_count: 0,
@@ -385,6 +434,11 @@ impl<'a> Parser<'a> {
             last_unexpected_token_span: None,
             last_type_ascription: None,
             subparser_name,
+            capture_state: CaptureState {
+                capturing: Capturing::No,
+                replace_ranges: Vec::new(),
+                inner_attr_ranges: Default::default(),
+            },
         };
 
         // Make parser point to the first token.
@@ -394,21 +448,29 @@ impl<'a> Parser<'a> {
     }
 
     fn next_tok(&mut self, fallback_span: Span) -> (Token, Spacing) {
-        let (mut next, spacing) = if self.desugar_doc_comments {
-            self.token_cursor.next_desugared()
-        } else {
-            self.token_cursor.next()
-        };
-        self.token_cursor.num_next_calls += 1;
-        // We've retrieved an token from the underlying
-        // cursor, so we no longer need to worry about
-        // an unglued token. See `break_and_eat` for more details
-        self.token_cursor.append_unglued_token = None;
-        if next.span.is_dummy() {
-            // Tweak the location for better diagnostics, but keep syntactic context intact.
-            next.span = fallback_span.with_ctxt(next.span.ctxt());
+        loop {
+            let (mut next, spacing) = if self.desugar_doc_comments {
+                self.token_cursor.next_desugared()
+            } else {
+                self.token_cursor.next()
+            };
+            self.token_cursor.num_next_calls += 1;
+            // We've retrieved an token from the underlying
+            // cursor, so we no longer need to worry about
+            // an unglued token. See `break_and_eat` for more details
+            self.token_cursor.break_last_token = false;
+            if next.span.is_dummy() {
+                // Tweak the location for better diagnostics, but keep syntactic context intact.
+                next.span = fallback_span.with_ctxt(next.span.ctxt());
+            }
+            if matches!(
+                next.kind,
+                token::OpenDelim(token::NoDelim) | token::CloseDelim(token::NoDelim)
+            ) {
+                continue;
+            }
+            return (next, spacing);
         }
-        (next, spacing)
     }
 
     pub fn unexpected<T>(&mut self) -> PResult<'a, T> {
@@ -606,8 +668,7 @@ impl<'a> Parser<'a> {
                 // If we consume any additional tokens, then this token
                 // is not needed (we'll capture the entire 'glued' token),
                 // and `next_tok` will set this field to `None`
-                self.token_cursor.append_unglued_token =
-                    Some((TokenTree::Token(self.token.clone()), Spacing::Alone));
+                self.token_cursor.break_last_token = true;
                 // Use the spacing of the glued token as the spacing
                 // of the unglued second token.
                 self.bump_with((Token::new(second, second_span), self.token_spacing));
@@ -688,6 +749,8 @@ impl<'a> Parser<'a> {
         let mut recovered = false;
         let mut trailing = false;
         let mut v = vec![];
+        let unclosed_delims = !self.unclosed_delims.is_empty();
+
         while !self.expect_any_with_type(kets, expect) {
             if let token::CloseDelim(..) | token::Eof = self.token.kind {
                 break;
@@ -708,7 +771,7 @@ impl<'a> Parser<'a> {
 
                             // Attempt to keep parsing if it was a similar separator.
                             if let Some(ref tokens) = t.similar_tokens() {
-                                if tokens.contains(&self.token.kind) {
+                                if tokens.contains(&self.token.kind) && !unclosed_delims {
                                     self.bump();
                                 }
                             }
@@ -867,15 +930,38 @@ impl<'a> Parser<'a> {
         }
 
         let frame = &self.token_cursor.frame;
-        match frame.tree_cursor.look_ahead(dist - 1) {
-            Some(tree) => match tree {
-                TokenTree::Token(token) => looker(token),
-                TokenTree::Delimited(dspan, delim, _) => {
-                    looker(&Token::new(token::OpenDelim(*delim), dspan.open))
-                }
-            },
-            None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)),
+        if frame.delim != DelimToken::NoDelim {
+            let all_normal = (0..dist).all(|i| {
+                let token = frame.tree_cursor.look_ahead(i);
+                !matches!(token, Some(TokenTree::Delimited(_, DelimToken::NoDelim, _)))
+            });
+            if all_normal {
+                return match frame.tree_cursor.look_ahead(dist - 1) {
+                    Some(tree) => match tree {
+                        TokenTree::Token(token) => looker(token),
+                        TokenTree::Delimited(dspan, delim, _) => {
+                            looker(&Token::new(token::OpenDelim(*delim), dspan.open))
+                        }
+                    },
+                    None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)),
+                };
+            }
         }
+
+        let mut cursor = self.token_cursor.clone();
+        let mut i = 0;
+        let mut token = Token::dummy();
+        while i < dist {
+            token = cursor.next().0;
+            if matches!(
+                token.kind,
+                token::OpenDelim(token::NoDelim) | token::CloseDelim(token::NoDelim)
+            ) {
+                continue;
+            }
+            i += 1;
+        }
+        return looker(&token);
     }
 
     /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.
@@ -987,7 +1073,7 @@ impl<'a> Parser<'a> {
                     }
 
                     // Collect tokens because they are used during lowering to HIR.
-                    let expr = self.collect_tokens_no_attrs(|this| this.parse_expr())?;
+                    let expr = self.parse_expr_force_collect()?;
                     let span = expr.span;
 
                     match &expr.kind {
@@ -1287,3 +1373,24 @@ pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedBrace>, sess: &Pa
         }
     }
 }
+
+/// A helper struct used when building a `AttrAnnotatedTokenStream` from
+/// a `LazyTokenStream`. Both delimiter and non-delimited tokens
+/// are stored as `FlatToken::Token`. A vector of `FlatToken`s
+/// is then 'parsed' to build up a `AttrAnnotatedTokenStream` with nested
+/// `AttrAnnotatedTokenTree::Delimited` tokens
+#[derive(Debug, Clone)]
+pub enum FlatToken {
+    /// A token - this holds both delimiter (e.g. '{' and '}')
+    /// and non-delimiter tokens
+    Token(Token),
+    /// Holds the `AttributesData` for an AST node. The
+    /// `AttributesData` is inserted directly into the
+    /// constructed `AttrAnnotatedTokenStream` as
+    /// a `AttrAnnotatedTokenTree::Attributes`
+    AttrTarget(AttributesData),
+    /// A special 'empty' token that is ignored during the conversion
+    /// to a `AttrAnnotatedTokenStream`. This is used to simplify the
+    /// handling of replace ranges.
+    Empty,
+}
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
index a84ae515144..5c4a2785d6e 100644
--- a/compiler/rustc_parse/src/parser/nonterminal.rs
+++ b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -4,7 +4,7 @@ use rustc_ast_pretty::pprust;
 use rustc_errors::PResult;
 use rustc_span::symbol::{kw, Ident};
 
-use crate::parser::pat::{GateOr, RecoverComma};
+use crate::parser::pat::RecoverComma;
 use crate::parser::{FollowedByType, ForceCollect, Parser, PathStyle};
 
 impl<'a> Parser<'a> {
@@ -61,7 +61,7 @@ impl<'a> Parser<'a> {
                 },
                 _ => false,
             },
-            NonterminalKind::Pat2018 { .. } | NonterminalKind::Pat2021 { .. } => match token.kind {
+            NonterminalKind::Pat2015 { .. } | NonterminalKind::Pat2021 { .. } => match token.kind {
                 token::Ident(..) |                  // box, ref, mut, and other identifiers (can stricten)
                 token::OpenDelim(token::Paren) |    // tuple pattern
                 token::OpenDelim(token::Bracket) |  // slice pattern
@@ -118,32 +118,17 @@ impl<'a> Parser<'a> {
                     return Err(self.struct_span_err(self.token.span, "expected a statement"));
                 }
             },
-            NonterminalKind::Pat2018 { .. } | NonterminalKind::Pat2021 { .. } => {
+            NonterminalKind::Pat2015 { .. } | NonterminalKind::Pat2021 { .. } => {
                 token::NtPat(self.collect_tokens_no_attrs(|this| match kind {
-                    NonterminalKind::Pat2018 { .. } => this.parse_pat_no_top_alt(None),
+                    NonterminalKind::Pat2015 { .. } => this.parse_pat_no_top_alt(None),
                     NonterminalKind::Pat2021 { .. } => {
-                        this.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::No)
+                        this.parse_pat_allow_top_alt(None, RecoverComma::No)
                     }
                     _ => unreachable!(),
                 })?)
             }
 
-            // If there are attributes present, then `parse_expr` will end up collecting tokens,
-            // turning the outer `collect_tokens_no_attrs` into a no-op due to the already present
-            // tokens. If there are *not* attributes present, then the outer
-            // `collect_tokens_no_attrs` will ensure that we will end up collecting tokens for the
-            // expressions.
-            //
-            // This is less efficient than it could be, since the outer `collect_tokens_no_attrs`
-            // still needs to snapshot the `TokenCursor` before calling `parse_expr`, even when
-            // `parse_expr` will end up collecting tokens. Ideally, this would work more like
-            // `parse_item`, and take in a `ForceCollect` parameter. However, this would require
-            // adding a `ForceCollect` parameter in a bunch of places in expression parsing
-            // for little gain. If the perf impact from this turns out to be noticeable, we should
-            // revisit this apporach.
-            NonterminalKind::Expr => {
-                token::NtExpr(self.collect_tokens_no_attrs(|this| this.parse_expr())?)
-            }
+            NonterminalKind::Expr => token::NtExpr(self.parse_expr_force_collect()?),
             NonterminalKind::Literal => {
                 // The `:literal` matcher does not support attributes
                 token::NtLiteral(
@@ -168,9 +153,7 @@ impl<'a> Parser<'a> {
             NonterminalKind::Path => token::NtPath(
                 self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?,
             ),
-            NonterminalKind::Meta => {
-                token::NtMeta(P(self.collect_tokens_no_attrs(|this| this.parse_attr_item(false))?))
-            }
+            NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item(true)?)),
             NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
             NonterminalKind::Vis => token::NtVis(
                 self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?,
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index 9e2e7359ca9..0abefbd6a12 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -3,7 +3,7 @@ use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
 use rustc_ast::mut_visit::{noop_visit_pat, MutVisitor};
 use rustc_ast::ptr::P;
 use rustc_ast::token;
-use rustc_ast::{self as ast, AttrVec, Attribute, FieldPat, MacCall, Pat, PatKind, RangeEnd};
+use rustc_ast::{self as ast, AttrVec, Attribute, MacCall, Pat, PatField, PatKind, RangeEnd};
 use rustc_ast::{BindingMode, Expr, ExprKind, Mutability, Path, QSelf, RangeSyntax};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, PResult};
@@ -17,13 +17,6 @@ pub(super) const PARAM_EXPECTED: Expected = Some("parameter name");
 
 const WHILE_PARSING_OR_MSG: &str = "while parsing this or-pattern starting here";
 
-/// Whether or not an or-pattern should be gated when occurring in the current context.
-#[derive(PartialEq, Clone, Copy)]
-pub enum GateOr {
-    Yes,
-    No,
-}
-
 /// Whether or not to recover a `,` when parsing or-patterns.
 #[derive(PartialEq, Copy, Clone)]
 pub enum RecoverComma {
@@ -31,6 +24,18 @@ pub enum RecoverComma {
     No,
 }
 
+/// The result of `eat_or_separator`. We want to distinguish which case we are in to avoid
+/// emitting duplicate diagnostics.
+#[derive(Debug, Clone, Copy)]
+enum EatOrResult {
+    /// We recovered from a trailing vert.
+    TrailingVert,
+    /// We ate an `|` (or `||` and recovered).
+    AteOr,
+    /// We did not eat anything (i.e. the current token is not `|` or `||`).
+    None,
+}
+
 impl<'a> Parser<'a> {
     /// Parses a pattern.
     ///
@@ -52,16 +57,31 @@ impl<'a> Parser<'a> {
     pub fn parse_pat_allow_top_alt(
         &mut self,
         expected: Expected,
-        gate_or: GateOr,
         rc: RecoverComma,
     ) -> PResult<'a, P<Pat>> {
+        self.parse_pat_allow_top_alt_inner(expected, rc).map(|(pat, _)| pat)
+    }
+
+    /// Returns the pattern and a bool indicating whether we recovered from a trailing vert (true =
+    /// recovered).
+    fn parse_pat_allow_top_alt_inner(
+        &mut self,
+        expected: Expected,
+        rc: RecoverComma,
+    ) -> PResult<'a, (P<Pat>, bool)> {
+        // Keep track of whether we recovered from a trailing vert so that we can avoid duplicated
+        // suggestions (which bothers rustfix).
+        //
         // Allow a '|' before the pats (RFCs 1925, 2530, and 2535).
-        let leading_vert_span =
-            if self.eat_or_separator(None) { Some(self.prev_token.span) } else { None };
+        let (leading_vert_span, mut trailing_vert) = match self.eat_or_separator(None) {
+            EatOrResult::AteOr => (Some(self.prev_token.span), false),
+            EatOrResult::TrailingVert => (None, true),
+            EatOrResult::None => (None, false),
+        };
 
         // Parse the first pattern (`p_0`).
         let first_pat = self.parse_pat_no_top_alt(expected)?;
-        self.maybe_recover_unexpected_comma(first_pat.span, rc, gate_or)?;
+        self.maybe_recover_unexpected_comma(first_pat.span, rc)?;
 
         // If the next token is not a `|`,
         // this is not an or-pattern and we should exit here.
@@ -70,46 +90,92 @@ impl<'a> Parser<'a> {
             // then we should really gate the leading `|`.
             // This complicated procedure is done purely for diagnostics UX.
             if let Some(leading_vert_span) = leading_vert_span {
-                if gate_or == GateOr::Yes && self.sess.gated_spans.is_ungated(sym::or_patterns) {
-                    self.sess.gated_spans.gate(sym::or_patterns, leading_vert_span);
-                }
-
                 // If there was a leading vert, treat this as an or-pattern. This improves
                 // diagnostics.
                 let span = leading_vert_span.to(self.prev_token.span);
-                return Ok(self.mk_pat(span, PatKind::Or(vec![first_pat])));
+                return Ok((self.mk_pat(span, PatKind::Or(vec![first_pat])), trailing_vert));
             }
 
-            return Ok(first_pat);
+            return Ok((first_pat, trailing_vert));
         }
 
         // Parse the patterns `p_1 | ... | p_n` where `n > 0`.
         let lo = leading_vert_span.unwrap_or(first_pat.span);
         let mut pats = vec![first_pat];
-        while self.eat_or_separator(Some(lo)) {
+        loop {
+            match self.eat_or_separator(Some(lo)) {
+                EatOrResult::AteOr => {}
+                EatOrResult::None => break,
+                EatOrResult::TrailingVert => {
+                    trailing_vert = true;
+                    break;
+                }
+            }
             let pat = self.parse_pat_no_top_alt(expected).map_err(|mut err| {
                 err.span_label(lo, WHILE_PARSING_OR_MSG);
                 err
             })?;
-            self.maybe_recover_unexpected_comma(pat.span, rc, gate_or)?;
+            self.maybe_recover_unexpected_comma(pat.span, rc)?;
             pats.push(pat);
         }
         let or_pattern_span = lo.to(self.prev_token.span);
 
-        // Feature gate the or-pattern if instructed:
-        if gate_or == GateOr::Yes {
-            self.sess.gated_spans.gate(sym::or_patterns, or_pattern_span);
+        Ok((self.mk_pat(or_pattern_span, PatKind::Or(pats)), trailing_vert))
+    }
+
+    /// Parse a pattern and (maybe) a `Colon` in positions where a pattern may be followed by a
+    /// type annotation (e.g. for `let` bindings or `fn` params).
+    ///
+    /// Generally, this corresponds to `pat_no_top_alt` followed by an optional `Colon`. It will
+    /// eat the `Colon` token if one is present.
+    ///
+    /// The return value represents the parsed pattern and `true` if a `Colon` was parsed (`false`
+    /// otherwise).
+    pub(super) fn parse_pat_before_ty(
+        &mut self,
+        expected: Expected,
+        rc: RecoverComma,
+        syntax_loc: &str,
+    ) -> PResult<'a, (P<Pat>, bool)> {
+        // We use `parse_pat_allow_top_alt` regardless of whether we actually want top-level
+        // or-patterns so that we can detect when a user tries to use it. This allows us to print a
+        // better error message.
+        let (pat, trailing_vert) = self.parse_pat_allow_top_alt_inner(expected, rc)?;
+        let colon = self.eat(&token::Colon);
+
+        if let PatKind::Or(pats) = &pat.kind {
+            let msg = format!("top-level or-patterns are not allowed in {}", syntax_loc);
+            let (help, fix) = if pats.len() == 1 {
+                // If all we have is a leading vert, then print a special message. This is the case
+                // if `parse_pat_allow_top_alt` returns an or-pattern with one variant.
+                let msg = "remove the `|`";
+                let fix = pprust::pat_to_string(&pat);
+                (msg, fix)
+            } else {
+                let msg = "wrap the pattern in parentheses";
+                let fix = format!("({})", pprust::pat_to_string(&pat));
+                (msg, fix)
+            };
+
+            if trailing_vert {
+                // We already emitted an error and suggestion to remove the trailing vert. Don't
+                // emit again.
+                self.sess.span_diagnostic.delay_span_bug(pat.span, &msg);
+            } else {
+                self.struct_span_err(pat.span, &msg)
+                    .span_suggestion(pat.span, help, fix, Applicability::MachineApplicable)
+                    .emit();
+            }
         }
 
-        Ok(self.mk_pat(or_pattern_span, PatKind::Or(pats)))
+        Ok((pat, colon))
     }
 
-    /// Parse the pattern for a function or function pointer parameter.
-    pub(super) fn parse_fn_param_pat(&mut self) -> PResult<'a, P<Pat>> {
-        // We actually do _not_ allow top-level or-patterns in function params, but we use
-        // `parse_pat_allow_top_alt` anyway so that we can detect when a user tries to use it. This
-        // allows us to print a better error message.
-        //
+    /// Parse the pattern for a function or function pointer parameter, followed by a colon.
+    ///
+    /// The return value represents the parsed pattern and `true` if a `Colon` was parsed (`false`
+    /// otherwise).
+    pub(super) fn parse_fn_param_pat_colon(&mut self) -> PResult<'a, (P<Pat>, bool)> {
         // In order to get good UX, we first recover in the case of a leading vert for an illegal
         // top-level or-pat. Normally, this means recovering both `|` and `||`, but in this case,
         // a leading `||` probably doesn't indicate an or-pattern attempt, so we handle that
@@ -128,53 +194,23 @@ impl<'a> Parser<'a> {
             self.bump();
         }
 
-        let pat = self.parse_pat_allow_top_alt(PARAM_EXPECTED, GateOr::No, RecoverComma::No)?;
-
-        if let PatKind::Or(..) = &pat.kind {
-            self.ban_illegal_fn_param_or_pat(&pat);
-        }
-
-        Ok(pat)
-    }
-
-    /// Ban `A | B` immediately in a parameter pattern and suggest wrapping in parens.
-    fn ban_illegal_fn_param_or_pat(&self, pat: &Pat) {
-        // If all we have a leading vert, then print a special message. This is the case if
-        // `parse_pat_allow_top_alt` returns an or-pattern with one variant.
-        let (msg, fix) = match &pat.kind {
-            PatKind::Or(pats) if pats.len() == 1 => {
-                let msg = "remove the leading `|`";
-                let fix = pprust::pat_to_string(pat);
-                (msg, fix)
-            }
-
-            _ => {
-                let msg = "wrap the pattern in parentheses";
-                let fix = format!("({})", pprust::pat_to_string(pat));
-                (msg, fix)
-            }
-        };
-
-        self.struct_span_err(pat.span, "an or-pattern parameter must be wrapped in parentheses")
-            .span_suggestion(pat.span, msg, fix, Applicability::MachineApplicable)
-            .emit();
+        self.parse_pat_before_ty(PARAM_EXPECTED, RecoverComma::No, "function parameters")
     }
 
     /// Eat the or-pattern `|` separator.
     /// If instead a `||` token is encountered, recover and pretend we parsed `|`.
-    fn eat_or_separator(&mut self, lo: Option<Span>) -> bool {
+    fn eat_or_separator(&mut self, lo: Option<Span>) -> EatOrResult {
         if self.recover_trailing_vert(lo) {
-            return false;
-        }
-
-        match self.token.kind {
-            token::OrOr => {
-                // Found `||`; Recover and pretend we parsed `|`.
-                self.ban_unexpected_or_or(lo);
-                self.bump();
-                true
-            }
-            _ => self.eat(&token::BinOp(token::Or)),
+            EatOrResult::TrailingVert
+        } else if matches!(self.token.kind, token::OrOr) {
+            // Found `||`; Recover and pretend we parsed `|`.
+            self.ban_unexpected_or_or(lo);
+            self.bump();
+            EatOrResult::AteOr
+        } else if self.eat(&token::BinOp(token::Or)) {
+            EatOrResult::AteOr
+        } else {
+            EatOrResult::None
         }
     }
 
@@ -190,14 +226,14 @@ impl<'a> Parser<'a> {
             matches!(
                 &token.uninterpolate().kind,
                 token::FatArrow // e.g. `a | => 0,`.
-            | token::Ident(kw::If, false) // e.g. `a | if expr`.
-            | token::Eq // e.g. `let a | = 0`.
-            | token::Semi // e.g. `let a |;`.
-            | token::Colon // e.g. `let a | :`.
-            | token::Comma // e.g. `let (a |,)`.
-            | token::CloseDelim(token::Bracket) // e.g. `let [a | ]`.
-            | token::CloseDelim(token::Paren) // e.g. `let (a | )`.
-            | token::CloseDelim(token::Brace) // e.g. `let A { f: a | }`.
+                | token::Ident(kw::If, false) // e.g. `a | if expr`.
+                | token::Eq // e.g. `let a | = 0`.
+                | token::Semi // e.g. `let a |;`.
+                | token::Colon // e.g. `let a | :`.
+                | token::Comma // e.g. `let (a |,)`.
+                | token::CloseDelim(token::Bracket) // e.g. `let [a | ]`.
+                | token::CloseDelim(token::Paren) // e.g. `let (a | )`.
+                | token::CloseDelim(token::Brace) // e.g. `let A { f: a | }`.
             )
         });
         match (is_end_ahead, &self.token.kind) {
@@ -227,12 +263,7 @@ impl<'a> Parser<'a> {
 
     /// Some special error handling for the "top-level" patterns in a match arm,
     /// `for` loop, `let`, &c. (in contrast to subpatterns within such).
-    fn maybe_recover_unexpected_comma(
-        &mut self,
-        lo: Span,
-        rc: RecoverComma,
-        gate_or: GateOr,
-    ) -> PResult<'a, ()> {
+    fn maybe_recover_unexpected_comma(&mut self, lo: Span, rc: RecoverComma) -> PResult<'a, ()> {
         if rc == RecoverComma::No || self.token != token::Comma {
             return Ok(());
         }
@@ -253,22 +284,18 @@ impl<'a> Parser<'a> {
         if let Ok(seq_snippet) = self.span_to_snippet(seq_span) {
             const MSG: &str = "try adding parentheses to match on a tuple...";
 
-            let or_suggestion =
-                gate_or == GateOr::No || !self.sess.gated_spans.is_ungated(sym::or_patterns);
             err.span_suggestion(
                 seq_span,
-                if or_suggestion { MSG } else { MSG.trim_end_matches('.') },
+                MSG,
                 format!("({})", seq_snippet),
                 Applicability::MachineApplicable,
             );
-            if or_suggestion {
-                err.span_suggestion(
-                    seq_span,
-                    "...or a vertical bar to match on multiple alternatives",
-                    seq_snippet.replace(",", " |"),
-                    Applicability::MachineApplicable,
-                );
-            }
+            err.span_suggestion(
+                seq_span,
+                "...or a vertical bar to match on multiple alternatives",
+                seq_snippet.replace(",", " |"),
+                Applicability::MachineApplicable,
+            );
         }
         Err(err)
     }
@@ -323,7 +350,7 @@ impl<'a> Parser<'a> {
         } else if self.check(&token::OpenDelim(token::Bracket)) {
             // Parse `[pat, pat,...]` as a slice pattern.
             let (pats, _) = self.parse_delim_comma_seq(token::Bracket, |p| {
-                p.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::No)
+                p.parse_pat_allow_top_alt(None, RecoverComma::No)
             })?;
             PatKind::Slice(pats)
         } else if self.check(&token::DotDot) && !self.is_pat_range_end_start(1) {
@@ -536,9 +563,8 @@ impl<'a> Parser<'a> {
 
     /// Parse a tuple or parenthesis pattern.
     fn parse_pat_tuple_or_parens(&mut self) -> PResult<'a, PatKind> {
-        let (fields, trailing_comma) = self.parse_paren_comma_seq(|p| {
-            p.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::No)
-        })?;
+        let (fields, trailing_comma) =
+            self.parse_paren_comma_seq(|p| p.parse_pat_allow_top_alt(None, RecoverComma::No))?;
 
         // Here, `(pat,)` is a tuple pattern.
         // For backward compatibility, `(..)` is a tuple pattern as well.
@@ -851,9 +877,8 @@ impl<'a> Parser<'a> {
         if qself.is_some() {
             return self.error_qpath_before_pat(&path, "(");
         }
-        let (fields, _) = self.parse_paren_comma_seq(|p| {
-            p.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::No)
-        })?;
+        let (fields, _) =
+            self.parse_paren_comma_seq(|p| p.parse_pat_allow_top_alt(None, RecoverComma::No))?;
         Ok(PatKind::TupleStruct(path, fields))
     }
 
@@ -868,7 +893,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses the fields of a struct-like pattern.
-    fn parse_pat_fields(&mut self) -> PResult<'a, (Vec<FieldPat>, bool)> {
+    fn parse_pat_fields(&mut self) -> PResult<'a, (Vec<PatField>, bool)> {
         let mut fields = Vec::new();
         let mut etc = false;
         let mut ate_comma = true;
@@ -1012,14 +1037,14 @@ impl<'a> Parser<'a> {
             .emit();
     }
 
-    fn parse_pat_field(&mut self, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, FieldPat> {
+    fn parse_pat_field(&mut self, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, PatField> {
         // Check if a colon exists one ahead. This means we're parsing a fieldname.
         let hi;
         let (subpat, fieldname, is_shorthand) = if self.look_ahead(1, |t| t == &token::Colon) {
             // Parsing a pattern of the form `fieldname: pat`.
             let fieldname = self.parse_field_name()?;
             self.bump();
-            let pat = self.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::No)?;
+            let pat = self.parse_pat_allow_top_alt(None, RecoverComma::No)?;
             hi = pat.span;
             (pat, fieldname, false)
         } else {
@@ -1044,7 +1069,7 @@ impl<'a> Parser<'a> {
             (subpat, fieldname, true)
         };
 
-        Ok(FieldPat {
+        Ok(PatField {
             ident: fieldname,
             pat: subpat,
             is_shorthand,
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index 07746f2390d..592f64f4a39 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -1,7 +1,7 @@
 use super::attr::DEFAULT_INNER_ATTR_FORBIDDEN;
 use super::diagnostics::{AttemptLocalParseRecovery, Error};
 use super::expr::LhsExpr;
-use super::pat::{GateOr, RecoverComma};
+use super::pat::RecoverComma;
 use super::path::PathStyle;
 use super::TrailingToken;
 use super::{AttrWrapper, BlockMode, ForceCollect, Parser, Restrictions, SemiColonMode};
@@ -13,7 +13,8 @@ use rustc_ast::token::{self, TokenKind};
 use rustc_ast::util::classify;
 use rustc_ast::AstLike;
 use rustc_ast::{AttrStyle, AttrVec, Attribute, MacCall, MacCallStmt, MacStmtStyle};
-use rustc_ast::{Block, BlockCheckMode, Expr, ExprKind, Local, Stmt, StmtKind, DUMMY_NODE_ID};
+use rustc_ast::{Block, BlockCheckMode, Expr, ExprKind, Local, Stmt};
+use rustc_ast::{StmtKind, DUMMY_NODE_ID};
 use rustc_errors::{Applicability, PResult};
 use rustc_span::source_map::{BytePos, Span};
 use rustc_span::symbol::{kw, sym};
@@ -34,7 +35,7 @@ impl<'a> Parser<'a> {
 
     /// If `force_capture` is true, forces collection of tokens regardless of whether
     /// or not we have attributes
-    fn parse_stmt_without_recovery(
+    crate fn parse_stmt_without_recovery(
         &mut self,
         capture_semi: bool,
         force_collect: ForceCollect,
@@ -47,39 +48,26 @@ impl<'a> Parser<'a> {
         if let token::Interpolated(nt) = &self.token.kind {
             if let token::NtStmt(stmt) = &**nt {
                 let mut stmt = stmt.clone();
-                return self.collect_tokens_trailing_token(
-                    attrs,
-                    force_collect,
-                    |this, mut attrs| {
-                        stmt.visit_attrs(|stmt_attrs| {
-                            mem::swap(stmt_attrs, &mut attrs);
-                            stmt_attrs.extend(attrs);
-                        });
-                        // Make sure we capture the token::Interpolated
-                        this.bump();
-                        Ok((Some(stmt), TrailingToken::None))
-                    },
-                );
+                self.bump();
+                stmt.visit_attrs(|stmt_attrs| {
+                    attrs.prepend_to_nt_inner(stmt_attrs);
+                });
+                return Ok(Some(stmt));
             }
         }
 
         Ok(Some(if self.token.is_keyword(kw::Let) {
             self.parse_local_mk(lo, attrs, capture_semi, force_collect)?
         } else if self.is_kw_followed_by_ident(kw::Mut) {
-            self.recover_stmt_local(
-                lo,
-                attrs.take_for_recovery().into(),
-                "missing keyword",
-                "let mut",
-            )?
+            self.recover_stmt_local(lo, attrs, "missing keyword", "let mut")?
         } else if self.is_kw_followed_by_ident(kw::Auto) {
             self.bump(); // `auto`
             let msg = "write `let` instead of `auto` to introduce a new variable";
-            self.recover_stmt_local(lo, attrs.take_for_recovery().into(), msg, "let")?
+            self.recover_stmt_local(lo, attrs, msg, "let")?
         } else if self.is_kw_followed_by_ident(sym::var) {
             self.bump(); // `var`
             let msg = "write `let` instead of `var` to introduce a new variable";
-            self.recover_stmt_local(lo, attrs.take_for_recovery().into(), msg, "let")?
+            self.recover_stmt_local(lo, attrs, msg, "let")?
         } else if self.check_path() && !self.token.is_qpath_start() && !self.is_path_start_item() {
             // We have avoided contextual keywords like `union`, items with `crate` visibility,
             // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
@@ -111,7 +99,7 @@ impl<'a> Parser<'a> {
         attrs: AttrWrapper,
         force_collect: ForceCollect,
     ) -> PResult<'a, Stmt> {
-        self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
+        let stmt = self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
             let path = this.parse_path(PathStyle::Expr)?;
 
             if this.eat(&token::Not) {
@@ -131,14 +119,22 @@ impl<'a> Parser<'a> {
             };
 
             let expr = this.with_res(Restrictions::STMT_EXPR, |this| {
-                let expr = this.parse_dot_or_call_expr_with(expr, lo, attrs)?;
+                this.parse_dot_or_call_expr_with(expr, lo, attrs)
+            })?;
+            // `DUMMY_SP` will get overwritten later in this function
+            Ok((this.mk_stmt(rustc_span::DUMMY_SP, StmtKind::Expr(expr)), TrailingToken::None))
+        })?;
+
+        if let StmtKind::Expr(expr) = stmt.kind {
+            // Perform this outside of the `collect_tokens_trailing_token` closure,
+            // since our outer attributes do not apply to this part of the expression
+            let expr = self.with_res(Restrictions::STMT_EXPR, |this| {
                 this.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(expr))
             })?;
-            Ok((
-                this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Expr(expr)),
-                TrailingToken::None,
-            ))
-        })
+            Ok(self.mk_stmt(lo.to(self.prev_token.span), StmtKind::Expr(expr)))
+        } else {
+            Ok(stmt)
+        }
     }
 
     /// Parses a statement macro `mac!(args)` provided a `path` representing `mac`.
@@ -182,7 +178,7 @@ impl<'a> Parser<'a> {
     fn recover_stmt_local(
         &mut self,
         lo: Span,
-        attrs: AttrVec,
+        attrs: AttrWrapper,
         msg: &str,
         sugg: &str,
     ) -> PResult<'a, Stmt> {
@@ -212,17 +208,23 @@ impl<'a> Parser<'a> {
         })
     }
 
-    fn recover_local_after_let(&mut self, lo: Span, attrs: AttrVec) -> PResult<'a, Stmt> {
-        let local = self.parse_local(attrs)?;
-        Ok(self.mk_stmt(lo.to(self.prev_token.span), StmtKind::Local(local)))
+    fn recover_local_after_let(&mut self, lo: Span, attrs: AttrWrapper) -> PResult<'a, Stmt> {
+        self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
+            let local = this.parse_local(attrs.into())?;
+            // FIXME - maybe capture semicolon in recovery?
+            Ok((
+                this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Local(local)),
+                TrailingToken::None,
+            ))
+        })
     }
 
     /// Parses a local variable declaration.
     fn parse_local(&mut self, attrs: AttrVec) -> PResult<'a, P<Local>> {
         let lo = self.prev_token.span;
-        let pat = self.parse_pat_allow_top_alt(None, GateOr::Yes, RecoverComma::Yes)?;
+        let (pat, colon) = self.parse_pat_before_ty(None, RecoverComma::Yes, "`let` bindings")?;
 
-        let (err, ty) = if self.eat(&token::Colon) {
+        let (err, ty) = if colon {
             // Save the state of the parser before parsing type normally, in case there is a `:`
             // instead of an `=` typo.
             let parser_snapshot_before_type = self.clone();
@@ -289,7 +291,7 @@ impl<'a> Parser<'a> {
         Ok(P(ast::Local { ty, pat, init, id: DUMMY_NODE_ID, span: lo.to(hi), attrs, tokens: None }))
     }
 
-    /// Parses the RHS of a local variable declaration (e.g., '= 14;').
+    /// Parses the RHS of a local variable declaration (e.g., `= 14;`).
     fn parse_initializer(&mut self, eq_optional: bool) -> PResult<'a, Option<P<Expr>>> {
         let eq_consumed = match self.token.kind {
             token::BinOpEq(..) => {
@@ -304,6 +306,7 @@ impl<'a> Parser<'a> {
                     "=".to_string(),
                     Applicability::MaybeIncorrect,
                 )
+                .help("if you meant to overwrite, remove the `let` binding")
                 .emit();
                 self.bump();
                 true
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index 8f03bfd4c3a..0f7b8ebd376 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -209,7 +209,7 @@ impl<'a> Parser<'a> {
         } else if self.eat_keyword(kw::Underscore) {
             // A type to be inferred `_`
             TyKind::Infer
-        } else if self.check_fn_front_matter() {
+        } else if self.check_fn_front_matter(false) {
             // Function pointer type
             self.parse_ty_bare_fn(lo, Vec::new(), recover_return_sign)?
         } else if self.check_keyword(kw::For) {
@@ -217,7 +217,7 @@ impl<'a> Parser<'a> {
             //   `for<'lt> [unsafe] [extern "ABI"] fn (&'lt S) -> T`
             //   `for<'lt> Trait1<'lt> + Trait2 + 'a`
             let lifetime_defs = self.parse_late_bound_lifetime_defs()?;
-            if self.check_fn_front_matter() {
+            if self.check_fn_front_matter(false) {
                 self.parse_ty_bare_fn(lo, lifetime_defs, recover_return_sign)?
             } else {
                 let path = self.parse_path(PathStyle::Type)?;
diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs
index 92d974690b5..50db69f4209 100644
--- a/compiler/rustc_parse_format/src/lib.rs
+++ b/compiler/rustc_parse_format/src/lib.rs
@@ -10,7 +10,7 @@
     test(attr(deny(warnings)))
 )]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(bool_to_option)]
 
 pub use Alignment::*;
@@ -213,11 +213,13 @@ impl<'a> Iterator for Parser<'a> {
                         Some(String(self.string(pos + 1)))
                     } else {
                         let arg = self.argument();
-                        if let Some(end) = self.must_consume('}') {
-                            let start = self.to_span_index(pos);
-                            let end = self.to_span_index(end + 1);
+                        if let Some(rbrace_byte_idx) = self.must_consume('}') {
+                            let lbrace_inner_offset = self.to_span_index(pos);
+                            let rbrace_inner_offset = self.to_span_index(rbrace_byte_idx);
                             if self.is_literal {
-                                self.arg_places.push(start.to(end));
+                                self.arg_places.push(
+                                    lbrace_inner_offset.to(InnerOffset(rbrace_inner_offset.0 + 1)),
+                                );
                             }
                         }
                         Some(NextArgument(arg))
@@ -735,25 +737,24 @@ fn find_skips_from_snippet(
     };
 
     fn find_skips(snippet: &str, is_raw: bool) -> Vec<usize> {
-        let mut eat_ws = false;
         let mut s = snippet.char_indices().peekable();
         let mut skips = vec![];
         while let Some((pos, c)) = s.next() {
             match (c, s.peek()) {
                 // skip whitespace and empty lines ending in '\\'
                 ('\\', Some((next_pos, '\n'))) if !is_raw => {
-                    eat_ws = true;
                     skips.push(pos);
                     skips.push(*next_pos);
                     let _ = s.next();
-                }
-                ('\\', Some((next_pos, '\n' | 'n' | 't'))) if eat_ws => {
-                    skips.push(pos);
-                    skips.push(*next_pos);
-                    let _ = s.next();
-                }
-                (' ' | '\n' | '\t', _) if eat_ws => {
-                    skips.push(pos);
+
+                    while let Some((pos, c)) = s.peek() {
+                        if matches!(c, ' ' | '\n' | '\t') {
+                            skips.push(*pos);
+                            let _ = s.next();
+                        } else {
+                            break;
+                        }
+                    }
                 }
                 ('\\', Some((next_pos, 'n' | 't' | 'r' | '0' | '\\' | '\'' | '\"'))) => {
                     skips.push(*next_pos);
@@ -804,10 +805,6 @@ fn find_skips_from_snippet(
                         }
                     }
                 }
-                _ if eat_ws => {
-                    // `take_while(|c| c.is_whitespace())`
-                    eat_ws = false;
-                }
                 _ => {}
             }
         }
diff --git a/compiler/rustc_passes/Cargo.toml b/compiler/rustc_passes/Cargo.toml
index c87799f1c2a..4069fb2127e 100644
--- a/compiler/rustc_passes/Cargo.toml
+++ b/compiler/rustc_passes/Cargo.toml
@@ -19,3 +19,4 @@ rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_lexer = { path = "../rustc_lexer" }
+rustc_ast_pretty = { path = "../rustc_ast_pretty" }
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 39245ea77e5..d57cba6420f 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -8,8 +8,8 @@ use rustc_middle::hir::map::Map;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::TyCtxt;
 
-use rustc_ast::{Attribute, LitKind, NestedMetaItem};
-use rustc_errors::{pluralize, struct_span_err};
+use rustc_ast::{Attribute, Lit, LitKind, NestedMetaItem};
+use rustc_errors::{pluralize, struct_span_err, Applicability};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
@@ -17,7 +17,9 @@ use rustc_hir::{
     self, FnSig, ForeignItem, ForeignItemKind, HirId, Item, ItemKind, TraitItem, CRATE_HIR_ID,
 };
 use rustc_hir::{MethodKind, Target};
-use rustc_session::lint::builtin::{CONFLICTING_REPR_HINTS, UNUSED_ATTRIBUTES};
+use rustc_session::lint::builtin::{
+    CONFLICTING_REPR_HINTS, INVALID_DOC_ATTRIBUTES, UNUSED_ATTRIBUTES,
+};
 use rustc_session::parse::feature_err;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
@@ -60,52 +62,55 @@ impl CheckAttrVisitor<'tcx> {
     fn check_attributes(
         &self,
         hir_id: HirId,
-        attrs: &'hir [Attribute],
         span: &Span,
         target: Target,
         item: Option<ItemLike<'_>>,
     ) {
         let mut is_valid = true;
+        let attrs = self.tcx.hir().attrs(hir_id);
         for attr in attrs {
-            is_valid &= if self.tcx.sess.check_name(attr, sym::inline) {
-                self.check_inline(hir_id, attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::non_exhaustive) {
-                self.check_non_exhaustive(hir_id, attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::marker) {
-                self.check_marker(hir_id, attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::target_feature) {
-                self.check_target_feature(hir_id, attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::track_caller) {
-                self.check_track_caller(hir_id, &attr.span, attrs, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::doc) {
-                self.check_doc_attrs(attr, hir_id, target)
-            } else if self.tcx.sess.check_name(attr, sym::no_link) {
-                self.check_no_link(hir_id, &attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::export_name) {
-                self.check_export_name(hir_id, &attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::rustc_args_required_const) {
-                self.check_rustc_args_required_const(&attr, span, target, item)
-            } else if self.tcx.sess.check_name(attr, sym::allow_internal_unstable) {
-                self.check_allow_internal_unstable(hir_id, &attr, span, target, &attrs)
-            } else if self.tcx.sess.check_name(attr, sym::rustc_allow_const_fn_unstable) {
-                self.check_rustc_allow_const_fn_unstable(hir_id, &attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::naked) {
-                self.check_naked(hir_id, attr, span, target)
-            } else if self.tcx.sess.check_name(attr, sym::rustc_legacy_const_generics) {
-                self.check_rustc_legacy_const_generics(&attr, span, target, item)
-            } else {
-                // lint-only checks
-                if self.tcx.sess.check_name(attr, sym::cold) {
-                    self.check_cold(hir_id, attr, span, target);
-                } else if self.tcx.sess.check_name(attr, sym::link_name) {
-                    self.check_link_name(hir_id, attr, span, target);
-                } else if self.tcx.sess.check_name(attr, sym::link_section) {
-                    self.check_link_section(hir_id, attr, span, target);
-                } else if self.tcx.sess.check_name(attr, sym::no_mangle) {
-                    self.check_no_mangle(hir_id, attr, span, target);
+            is_valid &= match attr.name_or_empty() {
+                sym::inline => self.check_inline(hir_id, attr, span, target),
+                sym::non_exhaustive => self.check_non_exhaustive(hir_id, attr, span, target),
+                sym::marker => self.check_marker(hir_id, attr, span, target),
+                sym::target_feature => self.check_target_feature(hir_id, attr, span, target),
+                sym::track_caller => {
+                    self.check_track_caller(hir_id, &attr.span, attrs, span, target)
                 }
-                true
+                sym::doc => self.check_doc_attrs(attr, hir_id, target),
+                sym::no_link => self.check_no_link(hir_id, &attr, span, target),
+                sym::export_name => self.check_export_name(hir_id, &attr, span, target),
+                sym::rustc_args_required_const => {
+                    self.check_rustc_args_required_const(&attr, span, target, item)
+                }
+                sym::rustc_layout_scalar_valid_range_start
+                | sym::rustc_layout_scalar_valid_range_end => {
+                    self.check_rustc_layout_scalar_valid_range(&attr, span, target)
+                }
+                sym::allow_internal_unstable => {
+                    self.check_allow_internal_unstable(hir_id, &attr, span, target, &attrs)
+                }
+                sym::rustc_allow_const_fn_unstable => {
+                    self.check_rustc_allow_const_fn_unstable(hir_id, &attr, span, target)
+                }
+                sym::naked => self.check_naked(hir_id, attr, span, target),
+                sym::rustc_legacy_const_generics => {
+                    self.check_rustc_legacy_const_generics(&attr, span, target, item)
+                }
+                sym::rustc_clean
+                | sym::rustc_dirty
+                | sym::rustc_if_this_changed
+                | sym::rustc_then_this_would_need => self.check_rustc_dirty_clean(&attr),
+                _ => true,
             };
+            // lint-only checks
+            match attr.name_or_empty() {
+                sym::cold => self.check_cold(hir_id, attr, span, target),
+                sym::link_name => self.check_link_name(hir_id, attr, span, target),
+                sym::link_section => self.check_link_section(hir_id, attr, span, target),
+                sym::no_mangle => self.check_no_mangle(hir_id, attr, span, target),
+                _ => {}
+            }
         }
 
         if !is_valid {
@@ -388,33 +393,50 @@ impl CheckAttrVisitor<'tcx> {
             .emit();
     }
 
-    fn check_doc_alias(&self, meta: &NestedMetaItem, hir_id: HirId, target: Target) -> bool {
-        let doc_alias = meta.value_str().map(|s| s.to_string()).unwrap_or_else(String::new);
+    fn check_doc_alias_value(
+        &self,
+        meta: &NestedMetaItem,
+        doc_alias: &str,
+        hir_id: HirId,
+        target: Target,
+        is_list: bool,
+    ) -> bool {
+        let tcx = self.tcx;
+        let err_fn = move |span: Span, msg: &str| {
+            tcx.sess.span_err(
+                span,
+                &format!(
+                    "`#[doc(alias{})]` {}",
+                    if is_list { "(\"...\")" } else { " = \"...\"" },
+                    msg,
+                ),
+            );
+            false
+        };
         if doc_alias.is_empty() {
-            self.doc_attr_str_error(meta, "alias");
-            return false;
+            return err_fn(
+                meta.name_value_literal_span().unwrap_or_else(|| meta.span()),
+                "attribute cannot have empty value",
+            );
         }
         if let Some(c) =
             doc_alias.chars().find(|&c| c == '"' || c == '\'' || (c.is_whitespace() && c != ' '))
         {
-            self.tcx
-                .sess
-                .struct_span_err(
-                    meta.name_value_literal_span().unwrap_or_else(|| meta.span()),
-                    &format!("{:?} character isn't allowed in `#[doc(alias = \"...\")]`", c),
-                )
-                .emit();
+            self.tcx.sess.span_err(
+                meta.name_value_literal_span().unwrap_or_else(|| meta.span()),
+                &format!(
+                    "{:?} character isn't allowed in `#[doc(alias{})]`",
+                    c,
+                    if is_list { "(\"...\")" } else { " = \"...\"" },
+                ),
+            );
             return false;
         }
         if doc_alias.starts_with(' ') || doc_alias.ends_with(' ') {
-            self.tcx
-                .sess
-                .struct_span_err(
-                    meta.name_value_literal_span().unwrap_or_else(|| meta.span()),
-                    "`#[doc(alias = \"...\")]` cannot start or end with ' '",
-                )
-                .emit();
-            return false;
+            return err_fn(
+                meta.name_value_literal_span().unwrap_or_else(|| meta.span()),
+                "cannot start or end with ' '",
+            );
         }
         if let Some(err) = match target {
             Target::Impl => Some("implementation block"),
@@ -440,27 +462,63 @@ impl CheckAttrVisitor<'tcx> {
             }
             _ => None,
         } {
-            self.tcx
-                .sess
-                .struct_span_err(
-                    meta.span(),
-                    &format!("`#[doc(alias = \"...\")]` isn't allowed on {}", err),
-                )
-                .emit();
-            return false;
+            return err_fn(meta.span(), &format!("isn't allowed on {}", err));
         }
         let item_name = self.tcx.hir().name(hir_id);
         if &*item_name.as_str() == doc_alias {
+            return err_fn(meta.span(), "is the same as the item's name");
+        }
+        true
+    }
+
+    fn check_doc_alias(&self, meta: &NestedMetaItem, hir_id: HirId, target: Target) -> bool {
+        if let Some(values) = meta.meta_item_list() {
+            let mut errors = 0;
+            for v in values {
+                match v.literal() {
+                    Some(l) => match l.kind {
+                        LitKind::Str(s, _) => {
+                            if !self.check_doc_alias_value(v, &s.as_str(), hir_id, target, true) {
+                                errors += 1;
+                            }
+                        }
+                        _ => {
+                            self.tcx
+                                .sess
+                                .struct_span_err(
+                                    v.span(),
+                                    "`#[doc(alias(\"a\"))]` expects string literals",
+                                )
+                                .emit();
+                            errors += 1;
+                        }
+                    },
+                    None => {
+                        self.tcx
+                            .sess
+                            .struct_span_err(
+                                v.span(),
+                                "`#[doc(alias(\"a\"))]` expects string literals",
+                            )
+                            .emit();
+                        errors += 1;
+                    }
+                }
+            }
+            errors == 0
+        } else if let Some(doc_alias) = meta.value_str().map(|s| s.to_string()) {
+            self.check_doc_alias_value(meta, &doc_alias, hir_id, target, false)
+        } else {
             self.tcx
                 .sess
                 .struct_span_err(
                     meta.span(),
-                    &format!("`#[doc(alias = \"...\")]` is the same as the item's name"),
+                    "doc alias attribute expects a string `#[doc(alias = \"a\")]` or a list of \
+                     strings `#[doc(alias(\"a\", \"b\"))]`",
                 )
                 .emit();
-            return false;
+            false
         }
-        true
     }
 
     fn check_doc_keyword(&self, meta: &NestedMetaItem, hir_id: HirId) -> bool {
@@ -518,7 +576,7 @@ impl CheckAttrVisitor<'tcx> {
                 .struct_span_err(
                     meta.span(),
                     &format!(
-                        "`#![doc({} = \"...\")]` isn't allowed as a crate level attribute",
+                        "`#![doc({} = \"...\")]` isn't allowed as a crate-level attribute",
                         attr_name,
                     ),
                 )
@@ -529,68 +587,109 @@ impl CheckAttrVisitor<'tcx> {
     }
 
     fn check_doc_attrs(&self, attr: &Attribute, hir_id: HirId, target: Target) -> bool {
-        if let Some(mi) = attr.meta() {
-            if let Some(list) = mi.meta_item_list() {
-                for meta in list {
-                    if meta.has_name(sym::alias) {
-                        if !self.check_attr_crate_level(meta, hir_id, "alias")
-                            || !self.check_doc_alias(meta, hir_id, target)
+        let mut is_valid = true;
+
+        if let Some(list) = attr.meta().and_then(|mi| mi.meta_item_list().map(|l| l.to_vec())) {
+            for meta in list {
+                if let Some(i_meta) = meta.meta_item() {
+                    match i_meta.name_or_empty() {
+                        sym::alias
+                            if !self.check_attr_crate_level(&meta, hir_id, "alias")
+                                || !self.check_doc_alias(&meta, hir_id, target) =>
                         {
-                            return false;
+                            is_valid = false
                         }
-                    } else if meta.has_name(sym::keyword) {
-                        if !self.check_attr_crate_level(meta, hir_id, "keyword")
-                            || !self.check_doc_keyword(meta, hir_id)
+
+                        sym::keyword
+                            if !self.check_attr_crate_level(&meta, hir_id, "keyword")
+                                || !self.check_doc_keyword(&meta, hir_id) =>
                         {
-                            return false;
+                            is_valid = false
                         }
-                    } else if let Some(i_meta) = meta.meta_item() {
-                        if ![
-                            sym::cfg,
-                            sym::hidden,
-                            sym::html_favicon_url,
-                            sym::html_logo_url,
-                            sym::html_no_source,
-                            sym::html_playground_url,
-                            sym::html_root_url,
-                            sym::include,
-                            sym::inline,
-                            sym::issue_tracker_base_url,
-                            sym::masked,
-                            sym::no_default_passes, // deprecated
-                            sym::no_inline,
-                            sym::passes, // deprecated
-                            sym::primitive,
-                            sym::spotlight,
-                            sym::test,
-                        ]
-                        .iter()
-                        .any(|m| i_meta.has_name(*m))
-                        {
+
+                        sym::test if CRATE_HIR_ID != hir_id => {
                             self.tcx.struct_span_lint_hir(
-                                UNUSED_ATTRIBUTES,
+                                INVALID_DOC_ATTRIBUTES,
                                 hir_id,
-                                i_meta.span,
+                                meta.span(),
                                 |lint| {
-                                    lint.build(&format!(
-                                        "unknown `doc` attribute `{}`",
-                                        i_meta.name_or_empty()
-                                    ))
-                                    .warn(
-                                        "this was previously accepted by the compiler but is \
-                                        being phased out; it will become a hard error in \
-                                        a future release!",
+                                    lint.build(
+                                        "`#![doc(test(...)]` is only allowed \
+                                         as a crate-level attribute",
                                     )
                                     .emit();
                                 },
                             );
-                            return false;
+                            is_valid = false;
+                        }
+
+                        // no_default_passes: deprecated
+                        // passes: deprecated
+                        // plugins: removed, but rustdoc warns about it itself
+                        sym::alias
+                        | sym::cfg
+                        | sym::hidden
+                        | sym::html_favicon_url
+                        | sym::html_logo_url
+                        | sym::html_no_source
+                        | sym::html_playground_url
+                        | sym::html_root_url
+                        | sym::include
+                        | sym::inline
+                        | sym::issue_tracker_base_url
+                        | sym::keyword
+                        | sym::masked
+                        | sym::no_default_passes
+                        | sym::no_inline
+                        | sym::notable_trait
+                        | sym::passes
+                        | sym::plugins
+                        | sym::primitive
+                        | sym::test => {}
+
+                        _ => {
+                            self.tcx.struct_span_lint_hir(
+                                INVALID_DOC_ATTRIBUTES,
+                                hir_id,
+                                i_meta.span,
+                                |lint| {
+                                    let mut diag = lint.build(&format!(
+                                        "unknown `doc` attribute `{}`",
+                                        rustc_ast_pretty::pprust::path_to_string(&i_meta.path),
+                                    ));
+                                    if i_meta.has_name(sym::spotlight) {
+                                        diag.note(
+                                            "`doc(spotlight)` was renamed to `doc(notable_trait)`",
+                                        );
+                                        diag.span_suggestion_short(
+                                            i_meta.span,
+                                            "use `notable_trait` instead",
+                                            String::from("notable_trait"),
+                                            Applicability::MachineApplicable,
+                                        );
+                                        diag.note("`doc(spotlight)` is now a no-op");
+                                    }
+                                    diag.emit();
+                                },
+                            );
+                            is_valid = false;
                         }
                     }
+                } else {
+                    self.tcx.struct_span_lint_hir(
+                        INVALID_DOC_ATTRIBUTES,
+                        hir_id,
+                        meta.span(),
+                        |lint| {
+                            lint.build(&format!("invalid `doc` attribute")).emit();
+                        },
+                    );
+                    is_valid = false;
                 }
             }
         }
-        true
+
+        is_valid
     }
 
     /// Checks if `#[cold]` is applied to a non-function. Returns `true` if valid.
@@ -794,6 +893,37 @@ impl CheckAttrVisitor<'tcx> {
         }
     }
 
+    fn check_rustc_layout_scalar_valid_range(
+        &self,
+        attr: &Attribute,
+        span: &Span,
+        target: Target,
+    ) -> bool {
+        if target != Target::Struct {
+            self.tcx
+                .sess
+                .struct_span_err(attr.span, "attribute should be applied to a struct")
+                .span_label(*span, "not a struct")
+                .emit();
+            return false;
+        }
+
+        let list = match attr.meta_item_list() {
+            None => return false,
+            Some(it) => it,
+        };
+
+        if matches!(&list[..], &[NestedMetaItem::Literal(Lit { kind: LitKind::Int(..), .. })]) {
+            true
+        } else {
+            self.tcx
+                .sess
+                .struct_span_err(attr.span, "expected exactly one integer literal argument")
+                .emit();
+            false
+        }
+    }
+
     /// Checks if `#[rustc_legacy_const_generics]` is applied to a function and has a valid argument.
     fn check_rustc_legacy_const_generics(
         &self,
@@ -893,6 +1023,20 @@ impl CheckAttrVisitor<'tcx> {
         }
     }
 
+    /// Checks that the dep-graph debugging attributes are only present when the query-dep-graph
+    /// option is passed to the compiler.
+    fn check_rustc_dirty_clean(&self, attr: &Attribute) -> bool {
+        if self.tcx.sess.opts.debugging_opts.query_dep_graph {
+            true
+        } else {
+            self.tcx
+                .sess
+                .struct_span_err(attr.span, "attribute requires -Z query-dep-graph to be enabled")
+                .emit();
+            false
+        }
+    }
+
     /// Checks if `#[link_section]` is applied to a function or static.
     fn check_link_section(&self, hir_id: HirId, attr: &Attribute, span: &Span, target: Target) {
         match target {
@@ -965,7 +1109,7 @@ impl CheckAttrVisitor<'tcx> {
         // ```
         let hints: Vec<_> = attrs
             .iter()
-            .filter(|attr| self.tcx.sess.check_name(attr, sym::repr))
+            .filter(|attr| attr.has_name(sym::repr))
             .filter_map(|attr| attr.meta_item_list())
             .flatten()
             .collect();
@@ -976,17 +1120,41 @@ impl CheckAttrVisitor<'tcx> {
         let mut is_transparent = false;
 
         for hint in &hints {
+            if !hint.is_meta_item() {
+                struct_span_err!(
+                    self.tcx.sess,
+                    hint.span(),
+                    E0565,
+                    "meta item in `repr` must be an identifier"
+                )
+                .emit();
+                continue;
+            }
+
             let (article, allowed_targets) = match hint.name_or_empty() {
-                _ if !matches!(target, Target::Struct | Target::Enum | Target::Union) => {
-                    ("a", "struct, enum, or union")
-                }
-                name @ sym::C | name @ sym::align => {
-                    is_c |= name == sym::C;
+                sym::C => {
+                    is_c = true;
                     match target {
                         Target::Struct | Target::Union | Target::Enum => continue,
                         _ => ("a", "struct, enum, or union"),
                     }
                 }
+                sym::align => {
+                    if let (Target::Fn, true) = (target, !self.tcx.features().fn_align) {
+                        feature_err(
+                            &self.tcx.sess.parse_sess,
+                            sym::fn_align,
+                            hint.span(),
+                            "`repr(align)` attributes on functions are unstable",
+                        )
+                        .emit();
+                    }
+
+                    match target {
+                        Target::Struct | Target::Union | Target::Enum | Target::Fn => continue,
+                        _ => ("a", "struct, enum, function, or union"),
+                    }
+                }
                 sym::packed => {
                     if target != Target::Struct && target != Target::Union {
                         ("a", "struct or union")
@@ -1043,7 +1211,17 @@ impl CheckAttrVisitor<'tcx> {
                         continue;
                     }
                 }
-                _ => continue,
+                _ => {
+                    struct_span_err!(
+                        self.tcx.sess,
+                        hint.span(),
+                        E0552,
+                        "unrecognized representation hint"
+                    )
+                    .emit();
+
+                    continue;
+                }
             };
 
             struct_span_err!(
@@ -1102,7 +1280,7 @@ impl CheckAttrVisitor<'tcx> {
 
     fn check_used(&self, attrs: &'hir [Attribute], target: Target) {
         for attr in attrs {
-            if self.tcx.sess.check_name(attr, sym::used) && target != Target::Static {
+            if attr.has_name(sym::used) && target != Target::Static {
                 self.tcx
                     .sess
                     .span_err(attr.span, "attribute must be applied to a `static` variable");
@@ -1200,53 +1378,29 @@ impl Visitor<'tcx> for CheckAttrVisitor<'tcx> {
 
     fn visit_item(&mut self, item: &'tcx Item<'tcx>) {
         let target = Target::from_item(item);
-        self.check_attributes(
-            item.hir_id(),
-            item.attrs,
-            &item.span,
-            target,
-            Some(ItemLike::Item(item)),
-        );
+        self.check_attributes(item.hir_id(), &item.span, target, Some(ItemLike::Item(item)));
         intravisit::walk_item(self, item)
     }
 
     fn visit_generic_param(&mut self, generic_param: &'tcx hir::GenericParam<'tcx>) {
         let target = Target::from_generic_param(generic_param);
-        self.check_attributes(
-            generic_param.hir_id,
-            generic_param.attrs,
-            &generic_param.span,
-            target,
-            None,
-        );
+        self.check_attributes(generic_param.hir_id, &generic_param.span, target, None);
         intravisit::walk_generic_param(self, generic_param)
     }
 
     fn visit_trait_item(&mut self, trait_item: &'tcx TraitItem<'tcx>) {
         let target = Target::from_trait_item(trait_item);
-        self.check_attributes(
-            trait_item.hir_id(),
-            &trait_item.attrs,
-            &trait_item.span,
-            target,
-            None,
-        );
+        self.check_attributes(trait_item.hir_id(), &trait_item.span, target, None);
         intravisit::walk_trait_item(self, trait_item)
     }
 
-    fn visit_struct_field(&mut self, struct_field: &'tcx hir::StructField<'tcx>) {
-        self.check_attributes(
-            struct_field.hir_id,
-            &struct_field.attrs,
-            &struct_field.span,
-            Target::Field,
-            None,
-        );
-        intravisit::walk_struct_field(self, struct_field);
+    fn visit_field_def(&mut self, struct_field: &'tcx hir::FieldDef<'tcx>) {
+        self.check_attributes(struct_field.hir_id, &struct_field.span, Target::Field, None);
+        intravisit::walk_field_def(self, struct_field);
     }
 
     fn visit_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) {
-        self.check_attributes(arm.hir_id, &arm.attrs, &arm.span, Target::Arm, None);
+        self.check_attributes(arm.hir_id, &arm.span, Target::Arm, None);
         intravisit::walk_arm(self, arm);
     }
 
@@ -1254,7 +1408,6 @@ impl Visitor<'tcx> for CheckAttrVisitor<'tcx> {
         let target = Target::from_foreign_item(f_item);
         self.check_attributes(
             f_item.hir_id(),
-            &f_item.attrs,
             &f_item.span,
             target,
             Some(ItemLike::ForeignItem(f_item)),
@@ -1264,14 +1417,14 @@ impl Visitor<'tcx> for CheckAttrVisitor<'tcx> {
 
     fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
         let target = target_from_impl_item(self.tcx, impl_item);
-        self.check_attributes(impl_item.hir_id(), &impl_item.attrs, &impl_item.span, target, None);
+        self.check_attributes(impl_item.hir_id(), &impl_item.span, target, None);
         intravisit::walk_impl_item(self, impl_item)
     }
 
     fn visit_stmt(&mut self, stmt: &'tcx hir::Stmt<'tcx>) {
         // When checking statements ignore expressions, they will be checked later.
         if let hir::StmtKind::Local(ref l) = stmt.kind {
-            self.check_attributes(l.hir_id, &l.attrs, &stmt.span, Target::Statement, None);
+            self.check_attributes(l.hir_id, &stmt.span, Target::Statement, None);
         }
         intravisit::walk_stmt(self, stmt)
     }
@@ -1282,7 +1435,7 @@ impl Visitor<'tcx> for CheckAttrVisitor<'tcx> {
             _ => Target::Expression,
         };
 
-        self.check_attributes(expr.hir_id, &expr.attrs, &expr.span, target, None);
+        self.check_attributes(expr.hir_id, &expr.span, target, None);
         intravisit::walk_expr(self, expr)
     }
 
@@ -1292,23 +1445,17 @@ impl Visitor<'tcx> for CheckAttrVisitor<'tcx> {
         generics: &'tcx hir::Generics<'tcx>,
         item_id: HirId,
     ) {
-        self.check_attributes(variant.id, variant.attrs, &variant.span, Target::Variant, None);
+        self.check_attributes(variant.id, &variant.span, Target::Variant, None);
         intravisit::walk_variant(self, variant, generics, item_id)
     }
 
     fn visit_macro_def(&mut self, macro_def: &'tcx hir::MacroDef<'tcx>) {
-        self.check_attributes(
-            macro_def.hir_id(),
-            macro_def.attrs,
-            &macro_def.span,
-            Target::MacroDef,
-            None,
-        );
+        self.check_attributes(macro_def.hir_id(), &macro_def.span, Target::MacroDef, None);
         intravisit::walk_macro_def(self, macro_def);
     }
 
     fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) {
-        self.check_attributes(param.hir_id, param.attrs, &param.span, Target::Param, None);
+        self.check_attributes(param.hir_id, &param.span, Target::Param, None);
 
         intravisit::walk_param(self, param);
     }
@@ -1335,7 +1482,7 @@ fn check_invalid_crate_level_attr(tcx: TyCtxt<'_>, attrs: &[Attribute]) {
         sym::path,
         sym::automatically_derived,
         sym::start,
-        sym::main,
+        sym::rustc_main,
     ];
 
     for attr in attrs {
@@ -1376,13 +1523,7 @@ fn check_mod_attrs(tcx: TyCtxt<'_>, module_def_id: LocalDefId) {
     tcx.hir().visit_exported_macros_in_krate(check_attr_visitor);
     check_invalid_macro_level_attr(tcx, tcx.hir().krate().non_exported_macro_attrs);
     if module_def_id.is_top_level_module() {
-        check_attr_visitor.check_attributes(
-            CRATE_HIR_ID,
-            tcx.hir().krate_attrs(),
-            &DUMMY_SP,
-            Target::Mod,
-            None,
-        );
+        check_attr_visitor.check_attributes(CRATE_HIR_ID, &DUMMY_SP, Target::Mod, None);
         check_invalid_crate_level_attr(tcx, tcx.hir().krate_attrs());
     }
 }
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index 62a95aa57c2..c63edf365a1 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -15,7 +15,6 @@ use rustc_middle::middle::privacy;
 use rustc_middle::ty::{self, DefIdTree, TyCtxt};
 use rustc_session::lint;
 
-use rustc_ast as ast;
 use rustc_span::symbol::{sym, Symbol};
 
 // Any local node that may call something in its body block should be
@@ -154,7 +153,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
         &mut self,
         lhs: &hir::Pat<'_>,
         res: Res,
-        pats: &[hir::FieldPat<'_>],
+        pats: &[hir::PatField<'_>],
     ) {
         let variant = match self.typeck_results().node_type(lhs.hir_id).kind() {
             ty::Adt(adt, _) => adt.variant_of_res(res),
@@ -225,7 +224,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
         self.inherited_pub_visibility = had_inherited_pub_visibility;
     }
 
-    fn mark_as_used_if_union(&mut self, adt: &ty::AdtDef, fields: &[hir::Field<'_>]) {
+    fn mark_as_used_if_union(&mut self, adt: &ty::AdtDef, fields: &[hir::ExprField<'_>]) {
         if adt.is_union() && adt.non_enum_variant().fields.len() > 1 && adt.did.is_local() {
             for field in fields {
                 let index = self.tcx.field_index(field.hir_id, self.typeck_results());
@@ -346,11 +345,8 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
     }
 }
 
-fn has_allow_dead_code_or_lang_attr(
-    tcx: TyCtxt<'_>,
-    id: hir::HirId,
-    attrs: &[ast::Attribute],
-) -> bool {
+fn has_allow_dead_code_or_lang_attr(tcx: TyCtxt<'_>, id: hir::HirId) -> bool {
+    let attrs = tcx.hir().attrs(id);
     if tcx.sess.contains_name(attrs, sym::lang) {
         return true;
     }
@@ -400,8 +396,7 @@ struct LifeSeeder<'k, 'tcx> {
 
 impl<'v, 'k, 'tcx> ItemLikeVisitor<'v> for LifeSeeder<'k, 'tcx> {
     fn visit_item(&mut self, item: &hir::Item<'_>) {
-        let allow_dead_code =
-            has_allow_dead_code_or_lang_attr(self.tcx, item.hir_id(), &item.attrs);
+        let allow_dead_code = has_allow_dead_code_or_lang_attr(self.tcx, item.hir_id());
         if allow_dead_code {
             self.worklist.push(item.hir_id());
         }
@@ -424,11 +419,7 @@ impl<'v, 'k, 'tcx> ItemLikeVisitor<'v> for LifeSeeder<'k, 'tcx> {
                 for impl_item_ref in items {
                     let impl_item = self.krate.impl_item(impl_item_ref.id);
                     if of_trait.is_some()
-                        || has_allow_dead_code_or_lang_attr(
-                            self.tcx,
-                            impl_item.hir_id(),
-                            &impl_item.attrs,
-                        )
+                        || has_allow_dead_code_or_lang_attr(self.tcx, impl_item.hir_id())
                     {
                         self.worklist.push(impl_item_ref.id.hir_id());
                     }
@@ -446,7 +437,7 @@ impl<'v, 'k, 'tcx> ItemLikeVisitor<'v> for LifeSeeder<'k, 'tcx> {
     fn visit_trait_item(&mut self, trait_item: &hir::TraitItem<'_>) {
         use hir::TraitItemKind::{Const, Fn};
         if matches!(trait_item.kind, Const(_, Some(_)) | Fn(_, hir::TraitFn::Provided(_)))
-            && has_allow_dead_code_or_lang_attr(self.tcx, trait_item.hir_id(), &trait_item.attrs)
+            && has_allow_dead_code_or_lang_attr(self.tcx, trait_item.hir_id())
         {
             self.worklist.push(trait_item.hir_id());
         }
@@ -459,11 +450,7 @@ impl<'v, 'k, 'tcx> ItemLikeVisitor<'v> for LifeSeeder<'k, 'tcx> {
     fn visit_foreign_item(&mut self, foreign_item: &hir::ForeignItem<'_>) {
         use hir::ForeignItemKind::{Fn, Static};
         if matches!(foreign_item.kind, Static(..) | Fn(..))
-            && has_allow_dead_code_or_lang_attr(
-                self.tcx,
-                foreign_item.hir_id(),
-                &foreign_item.attrs,
-            )
+            && has_allow_dead_code_or_lang_attr(self.tcx, foreign_item.hir_id())
         {
             self.worklist.push(foreign_item.hir_id());
         }
@@ -538,22 +525,21 @@ impl DeadVisitor<'tcx> {
         should_warn && !self.symbol_is_live(item.hir_id())
     }
 
-    fn should_warn_about_field(&mut self, field: &hir::StructField<'_>) -> bool {
+    fn should_warn_about_field(&mut self, field: &hir::FieldDef<'_>) -> bool {
         let field_type = self.tcx.type_of(self.tcx.hir().local_def_id(field.hir_id));
         !field.is_positional()
             && !self.symbol_is_live(field.hir_id)
             && !field_type.is_phantom_data()
-            && !has_allow_dead_code_or_lang_attr(self.tcx, field.hir_id, &field.attrs)
+            && !has_allow_dead_code_or_lang_attr(self.tcx, field.hir_id)
     }
 
     fn should_warn_about_variant(&mut self, variant: &hir::Variant<'_>) -> bool {
-        !self.symbol_is_live(variant.id)
-            && !has_allow_dead_code_or_lang_attr(self.tcx, variant.id, &variant.attrs)
+        !self.symbol_is_live(variant.id) && !has_allow_dead_code_or_lang_attr(self.tcx, variant.id)
     }
 
     fn should_warn_about_foreign_item(&mut self, fi: &hir::ForeignItem<'_>) -> bool {
         !self.symbol_is_live(fi.hir_id())
-            && !has_allow_dead_code_or_lang_attr(self.tcx, fi.hir_id(), &fi.attrs)
+            && !has_allow_dead_code_or_lang_attr(self.tcx, fi.hir_id())
     }
 
     // id := HIR id of an item's definition.
@@ -664,11 +650,11 @@ impl Visitor<'tcx> for DeadVisitor<'tcx> {
         intravisit::walk_foreign_item(self, fi);
     }
 
-    fn visit_struct_field(&mut self, field: &'tcx hir::StructField<'tcx>) {
+    fn visit_field_def(&mut self, field: &'tcx hir::FieldDef<'tcx>) {
         if self.should_warn_about_field(&field) {
             self.warn_dead_code(field.hir_id, field.span, field.ident.name, "read");
         }
-        intravisit::walk_struct_field(self, field);
+        intravisit::walk_field_def(self, field);
     }
 
     fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
diff --git a/compiler/rustc_passes/src/diagnostic_items.rs b/compiler/rustc_passes/src/diagnostic_items.rs
index 3ec7ea39248..8dd3700e5b6 100644
--- a/compiler/rustc_passes/src/diagnostic_items.rs
+++ b/compiler/rustc_passes/src/diagnostic_items.rs
@@ -27,19 +27,19 @@ struct DiagnosticItemCollector<'tcx> {
 
 impl<'v, 'tcx> ItemLikeVisitor<'v> for DiagnosticItemCollector<'tcx> {
     fn visit_item(&mut self, item: &hir::Item<'_>) {
-        self.observe_item(&item.attrs, item.def_id);
+        self.observe_item(item.def_id);
     }
 
     fn visit_trait_item(&mut self, trait_item: &hir::TraitItem<'_>) {
-        self.observe_item(&trait_item.attrs, trait_item.def_id);
+        self.observe_item(trait_item.def_id);
     }
 
     fn visit_impl_item(&mut self, impl_item: &hir::ImplItem<'_>) {
-        self.observe_item(&impl_item.attrs, impl_item.def_id);
+        self.observe_item(impl_item.def_id);
     }
 
     fn visit_foreign_item(&mut self, foreign_item: &hir::ForeignItem<'_>) {
-        self.observe_item(foreign_item.attrs, foreign_item.def_id);
+        self.observe_item(foreign_item.def_id);
     }
 }
 
@@ -48,7 +48,9 @@ impl<'tcx> DiagnosticItemCollector<'tcx> {
         DiagnosticItemCollector { tcx, items: Default::default() }
     }
 
-    fn observe_item(&mut self, attrs: &[ast::Attribute], def_id: LocalDefId) {
+    fn observe_item(&mut self, def_id: LocalDefId) {
+        let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+        let attrs = self.tcx.hir().attrs(hir_id);
         if let Some(name) = extract(&self.tcx.sess, attrs) {
             // insert into our table
             collect_item(self.tcx, &mut self.items, name, def_id.to_def_id());
@@ -105,7 +107,7 @@ fn collect<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap<Symbol, DefId> {
     tcx.hir().krate().visit_all_item_likes(&mut collector);
 
     for m in tcx.hir().krate().exported_macros {
-        collector.observe_item(m.attrs, m.def_id);
+        collector.observe_item(m.def_id);
     }
 
     collector.items
diff --git a/compiler/rustc_passes/src/entry.rs b/compiler/rustc_passes/src/entry.rs
index 0d3a7ea3a8a..e1b750df33c 100644
--- a/compiler/rustc_passes/src/entry.rs
+++ b/compiler/rustc_passes/src/entry.rs
@@ -2,7 +2,7 @@ use rustc_ast::entry::EntryPointType;
 use rustc_errors::struct_span_err;
 use rustc_hir::def_id::{CrateNum, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::itemlikevisit::ItemLikeVisitor;
-use rustc_hir::{ForeignItem, HirId, ImplItem, Item, ItemKind, TraitItem};
+use rustc_hir::{ForeignItem, HirId, ImplItem, Item, ItemKind, TraitItem, CRATE_HIR_ID};
 use rustc_middle::hir::map::Map;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::TyCtxt;
@@ -60,7 +60,7 @@ fn entry_fn(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option<(LocalDefId, EntryFnType)
     }
 
     // If the user wants no main function at all, then stop here.
-    if tcx.sess.contains_name(&tcx.hir().krate().item.attrs, sym::no_main) {
+    if tcx.sess.contains_name(&tcx.hir().attrs(CRATE_HIR_ID), sym::no_main) {
         return None;
     }
 
@@ -80,10 +80,11 @@ fn entry_fn(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option<(LocalDefId, EntryFnType)
 
 // Beware, this is duplicated in `librustc_builtin_macros/test_harness.rs`
 // (with `ast::Item`), so make sure to keep them in sync.
-fn entry_point_type(sess: &Session, item: &Item<'_>, at_root: bool) -> EntryPointType {
-    if sess.contains_name(&item.attrs, sym::start) {
+fn entry_point_type(ctxt: &EntryContext<'_, '_>, item: &Item<'_>, at_root: bool) -> EntryPointType {
+    let attrs = ctxt.map.attrs(item.hir_id());
+    if ctxt.session.contains_name(attrs, sym::start) {
         EntryPointType::Start
-    } else if sess.contains_name(&item.attrs, sym::main) {
+    } else if ctxt.session.contains_name(attrs, sym::rustc_main) {
         EntryPointType::MainAttr
     } else if item.ident.name == sym::main {
         if at_root {
@@ -103,14 +104,15 @@ fn throw_attr_err(sess: &Session, span: Span, attr: &str) {
 }
 
 fn find_item(item: &Item<'_>, ctxt: &mut EntryContext<'_, '_>, at_root: bool) {
-    match entry_point_type(&ctxt.session, item, at_root) {
+    match entry_point_type(ctxt, item, at_root) {
         EntryPointType::None => (),
         _ if !matches!(item.kind, ItemKind::Fn(..)) => {
-            if let Some(attr) = ctxt.session.find_by_name(item.attrs, sym::start) {
+            let attrs = ctxt.map.attrs(item.hir_id());
+            if let Some(attr) = ctxt.session.find_by_name(attrs, sym::start) {
                 throw_attr_err(&ctxt.session, attr.span, "start");
             }
-            if let Some(attr) = ctxt.session.find_by_name(item.attrs, sym::main) {
-                throw_attr_err(&ctxt.session, attr.span, "main");
+            if let Some(attr) = ctxt.session.find_by_name(attrs, sym::rustc_main) {
+                throw_attr_err(&ctxt.session, attr.span, "rustc_main");
             }
         }
         EntryPointType::MainNamed => {
@@ -169,7 +171,7 @@ fn configure_main(
 }
 
 fn no_main_err(tcx: TyCtxt<'_>, visitor: &EntryContext<'_, '_>) {
-    let sp = tcx.hir().krate().item.span;
+    let sp = tcx.hir().krate().item.inner;
     if *tcx.sess.parse_sess.reached_eof.borrow() {
         // There's an unclosed brace that made the parser reach `Eof`, we shouldn't complain about
         // the missing `fn main()` then as it might have been hidden inside an unclosed block.
@@ -191,10 +193,7 @@ fn no_main_err(tcx: TyCtxt<'_>, visitor: &EntryContext<'_, '_>) {
             err.span_note(span, "here is a function named `main`");
         }
         err.note("you have one or more functions named `main` not defined at the crate level");
-        err.help(
-            "either move the `main` function definitions or attach the `#[main]` attribute \
-                  to one of them",
-        );
+        err.help("consider moving the `main` function definitions");
         // There were some functions named `main` though. Try to give the user a hint.
         format!(
             "the main function must be defined at the crate level{}",
diff --git a/compiler/rustc_passes/src/hir_id_validator.rs b/compiler/rustc_passes/src/hir_id_validator.rs
index 79e3b5952ac..944a3097a61 100644
--- a/compiler/rustc_passes/src/hir_id_validator.rs
+++ b/compiler/rustc_passes/src/hir_id_validator.rs
@@ -172,17 +172,4 @@ impl<'a, 'hir> intravisit::Visitor<'hir> for HirIdValidator<'a, 'hir> {
         // we are currently in. So for those it's correct that they have a
         // different owner.
     }
-
-    fn visit_generic_param(&mut self, param: &'hir hir::GenericParam<'hir>) {
-        if let hir::GenericParamKind::Type {
-            synthetic: Some(hir::SyntheticTyParamKind::ImplTrait),
-            ..
-        } = param.kind
-        {
-            // Synthetic impl trait parameters are owned by the node of the desugared type.
-            // This means it is correct for them to have a different owner.
-        } else {
-            intravisit::walk_generic_param(self, param);
-        }
-    }
 }
diff --git a/compiler/rustc_passes/src/hir_stats.rs b/compiler/rustc_passes/src/hir_stats.rs
index a2b6dd17ad9..2bed8cadeb9 100644
--- a/compiler/rustc_passes/src/hir_stats.rs
+++ b/compiler/rustc_passes/src/hir_stats.rs
@@ -201,9 +201,9 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> {
         hir_visit::walk_param_bound(self, bounds)
     }
 
-    fn visit_struct_field(&mut self, s: &'v hir::StructField<'v>) {
-        self.record("StructField", Id::Node(s.hir_id), s);
-        hir_visit::walk_struct_field(self, s)
+    fn visit_field_def(&mut self, s: &'v hir::FieldDef<'v>) {
+        self.record("FieldDef", Id::Node(s.hir_id), s);
+        hir_visit::walk_field_def(self, s)
     }
 
     fn visit_variant(
@@ -241,7 +241,7 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> {
         hir_visit::walk_assoc_type_binding(self, type_binding)
     }
 
-    fn visit_attribute(&mut self, attr: &'v ast::Attribute) {
+    fn visit_attribute(&mut self, _: hir::HirId, attr: &'v ast::Attribute) {
         self.record("Attribute", Id::Attr(attr.id), attr);
     }
 
@@ -316,9 +316,9 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> {
         ast_visit::walk_param_bound(self, bounds)
     }
 
-    fn visit_struct_field(&mut self, s: &'v ast::StructField) {
-        self.record("StructField", Id::None, s);
-        ast_visit::walk_struct_field(self, s)
+    fn visit_field_def(&mut self, s: &'v ast::FieldDef) {
+        self.record("FieldDef", Id::None, s);
+        ast_visit::walk_field_def(self, s)
     }
 
     fn visit_variant(&mut self, v: &'v ast::Variant) {
diff --git a/compiler/rustc_passes/src/intrinsicck.rs b/compiler/rustc_passes/src/intrinsicck.rs
index 0f4bb635eee..3f095d0e824 100644
--- a/compiler/rustc_passes/src/intrinsicck.rs
+++ b/compiler/rustc_passes/src/intrinsicck.rs
@@ -347,7 +347,7 @@ impl ExprVisitor<'tcx> {
     }
 
     fn check_asm(&self, asm: &hir::InlineAsm<'tcx>) {
-        for (idx, (op, _op_sp)) in asm.operands.iter().enumerate() {
+        for (idx, (op, op_sp)) in asm.operands.iter().enumerate() {
             match *op {
                 hir::InlineAsmOperand::In { reg, ref expr } => {
                     self.check_asm_operand_type(idx, reg, expr, asm.template, None);
@@ -372,14 +372,15 @@ impl ExprVisitor<'tcx> {
                         );
                     }
                 }
-                hir::InlineAsmOperand::Const { ref expr } => {
-                    let ty = self.typeck_results.expr_ty_adjusted(expr);
-                    match ty.kind() {
+                hir::InlineAsmOperand::Const { ref anon_const } => {
+                    let anon_const_def_id = self.tcx.hir().local_def_id(anon_const.hir_id);
+                    let value = ty::Const::from_anon_const(self.tcx, anon_const_def_id);
+                    match value.ty.kind() {
                         ty::Int(_) | ty::Uint(_) | ty::Float(_) => {}
                         _ => {
                             let msg =
                                 "asm `const` arguments must be integer or floating-point values";
-                            self.tcx.sess.span_err(expr.span, msg);
+                            self.tcx.sess.span_err(*op_sp, msg);
                         }
                     }
                 }
diff --git a/compiler/rustc_passes/src/lang_items.rs b/compiler/rustc_passes/src/lang_items.rs
index 8e2ad7f783e..7e6bb9779f0 100644
--- a/compiler/rustc_passes/src/lang_items.rs
+++ b/compiler/rustc_passes/src/lang_items.rs
@@ -13,7 +13,6 @@ use crate::weak_lang_items;
 use rustc_middle::middle::cstore::ExternCrate;
 use rustc_middle::ty::TyCtxt;
 
-use rustc_ast::Attribute;
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
@@ -30,29 +29,21 @@ struct LanguageItemCollector<'tcx> {
 
 impl ItemLikeVisitor<'v> for LanguageItemCollector<'tcx> {
     fn visit_item(&mut self, item: &hir::Item<'_>) {
-        self.check_for_lang(Target::from_item(item), item.hir_id(), item.attrs);
+        self.check_for_lang(Target::from_item(item), item.hir_id());
 
         if let hir::ItemKind::Enum(def, ..) = &item.kind {
             for variant in def.variants {
-                self.check_for_lang(Target::Variant, variant.id, variant.attrs);
+                self.check_for_lang(Target::Variant, variant.id);
             }
         }
     }
 
     fn visit_trait_item(&mut self, trait_item: &hir::TraitItem<'_>) {
-        self.check_for_lang(
-            Target::from_trait_item(trait_item),
-            trait_item.hir_id(),
-            trait_item.attrs,
-        )
+        self.check_for_lang(Target::from_trait_item(trait_item), trait_item.hir_id())
     }
 
     fn visit_impl_item(&mut self, impl_item: &hir::ImplItem<'_>) {
-        self.check_for_lang(
-            target_from_impl_item(self.tcx, impl_item),
-            impl_item.hir_id(),
-            impl_item.attrs,
-        )
+        self.check_for_lang(target_from_impl_item(self.tcx, impl_item), impl_item.hir_id())
     }
 
     fn visit_foreign_item(&mut self, _: &hir::ForeignItem<'_>) {}
@@ -63,7 +54,8 @@ impl LanguageItemCollector<'tcx> {
         LanguageItemCollector { tcx, items: LanguageItems::new() }
     }
 
-    fn check_for_lang(&mut self, actual_target: Target, hir_id: HirId, attrs: &[Attribute]) {
+    fn check_for_lang(&mut self, actual_target: Target, hir_id: HirId) {
+        let attrs = self.tcx.hir().attrs(hir_id);
         let check_name = |attr, sym| self.tcx.sess.check_name(attr, sym);
         if let Some((value, span)) = extract(check_name, &attrs) {
             match ITEM_REFS.get(&value).cloned() {
diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs
index 9759a500e06..933e8ad1d72 100644
--- a/compiler/rustc_passes/src/lib.rs
+++ b/compiler/rustc_passes/src/lib.rs
@@ -9,8 +9,9 @@
 #![feature(const_panic)]
 #![feature(crate_visibility_modifier)]
 #![feature(in_band_lifetimes)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 #[macro_use]
diff --git a/compiler/rustc_passes/src/lib_features.rs b/compiler/rustc_passes/src/lib_features.rs
index 02b20e45d00..3dfe317a4bd 100644
--- a/compiler/rustc_passes/src/lib_features.rs
+++ b/compiler/rustc_passes/src/lib_features.rs
@@ -120,7 +120,7 @@ impl Visitor<'tcx> for LibFeatureCollector<'tcx> {
         NestedVisitorMap::All(self.tcx.hir())
     }
 
-    fn visit_attribute(&mut self, attr: &'tcx Attribute) {
+    fn visit_attribute(&mut self, _: rustc_hir::HirId, attr: &'tcx Attribute) {
         if let Some((feature, stable, span)) = self.extract(attr) {
             self.collect_feature(feature, stable, span);
         }
@@ -131,7 +131,7 @@ fn collect(tcx: TyCtxt<'_>) -> LibFeatures {
     let mut collector = LibFeatureCollector::new(tcx);
     let krate = tcx.hir().krate();
     for attr in krate.non_exported_macro_attrs {
-        collector.visit_attribute(attr);
+        collector.visit_attribute(rustc_hir::CRATE_HIR_ID, attr);
     }
     intravisit::walk_crate(&mut collector, krate);
     collector.lib_features
diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs
index a96f3323744..e22a108aaf0 100644
--- a/compiler/rustc_passes/src/liveness.rs
+++ b/compiler/rustc_passes/src/liveness.rs
@@ -95,7 +95,7 @@ use rustc_hir::{Expr, HirId, HirIdMap, HirIdSet};
 use rustc_index::vec::IndexVec;
 use rustc_middle::hir::map::Map;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, DefIdTree, TyCtxt};
+use rustc_middle::ty::{self, DefIdTree, RootVariableMinCaptureList, TyCtxt};
 use rustc_session::lint;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
@@ -103,6 +103,7 @@ use rustc_span::Span;
 use std::collections::VecDeque;
 use std::io;
 use std::io::prelude::*;
+use std::iter;
 use std::rc::Rc;
 
 mod rwu_table;
@@ -331,7 +332,7 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
             }
         }
 
-        if let Some(captures) = maps.tcx.typeck(local_def_id).closure_captures.get(&def_id) {
+        if let Some(captures) = maps.tcx.typeck(local_def_id).closure_min_captures.get(&def_id) {
             for &var_hir_id in captures.keys() {
                 let var_name = maps.tcx.hir().name(var_hir_id);
                 maps.add_variable(Upvar(var_hir_id, var_name));
@@ -408,10 +409,10 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
                 if let Some(captures) = self
                     .tcx
                     .typeck(closure_def_id)
-                    .closure_captures
+                    .closure_min_captures
                     .get(&closure_def_id.to_def_id())
                 {
-                    // If closure captures is Some, upvars_mentioned must also be Some
+                    // If closure_min_captures is Some, upvars_mentioned must also be Some
                     let upvars = self.tcx.upvars_mentioned(closure_def_id).unwrap();
                     call_caps.extend(captures.keys().map(|var_id| {
                         let upvar = upvars[var_id];
@@ -481,11 +482,10 @@ const ACC_USE: u32 = 4;
 
 struct Liveness<'a, 'tcx> {
     ir: &'a mut IrMaps<'tcx>,
-    body_owner: LocalDefId,
     typeck_results: &'a ty::TypeckResults<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     upvars: Option<&'tcx FxIndexMap<hir::HirId, hir::Upvar>>,
-    closure_captures: Option<&'tcx FxIndexMap<hir::HirId, ty::UpvarId>>,
+    closure_min_captures: Option<&'tcx RootVariableMinCaptureList<'tcx>>,
     successors: IndexVec<LiveNode, Option<LiveNode>>,
     rwu_table: rwu_table::RWUTable,
 
@@ -509,8 +509,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
         let typeck_results = ir.tcx.typeck(body_owner);
         let param_env = ir.tcx.param_env(body_owner);
         let upvars = ir.tcx.upvars_mentioned(body_owner);
-        let closure_captures = typeck_results.closure_captures.get(&body_owner.to_def_id());
-
+        let closure_min_captures = typeck_results.closure_min_captures.get(&body_owner.to_def_id());
         let closure_ln = ir.add_live_node(ClosureNode);
         let exit_ln = ir.add_live_node(ExitNode);
 
@@ -519,11 +518,10 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
         Liveness {
             ir,
-            body_owner,
             typeck_results,
             param_env,
             upvars,
-            closure_captures,
+            closure_min_captures,
             successors: IndexVec::from_elem_n(None, num_live_nodes),
             rwu_table: rwu_table::RWUTable::new(num_live_nodes, num_vars),
             closure_ln,
@@ -707,25 +705,27 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
         // if they are live on the entry to the closure, since only the closure
         // itself can access them on subsequent calls.
 
-        if let Some(closure_captures) = self.closure_captures {
+        if let Some(closure_min_captures) = self.closure_min_captures {
             // Mark upvars captured by reference as used after closure exits.
-            // Since closure_captures is Some, upvars must exists too.
-            let upvars = self.upvars.unwrap();
-            for (&var_hir_id, upvar_id) in closure_captures {
-                let upvar = upvars[&var_hir_id];
-                match self.typeck_results.upvar_capture(*upvar_id) {
-                    ty::UpvarCapture::ByRef(_) => {
-                        let var = self.variable(var_hir_id, upvar.span);
-                        self.acc(self.exit_ln, var, ACC_READ | ACC_USE);
+            for (&var_hir_id, min_capture_list) in closure_min_captures {
+                for captured_place in min_capture_list {
+                    match captured_place.info.capture_kind {
+                        ty::UpvarCapture::ByRef(_) => {
+                            let var = self.variable(
+                                var_hir_id,
+                                captured_place.get_capture_kind_span(self.ir.tcx),
+                            );
+                            self.acc(self.exit_ln, var, ACC_READ | ACC_USE);
+                        }
+                        ty::UpvarCapture::ByValue(_) => {}
                     }
-                    ty::UpvarCapture::ByValue(_) => {}
                 }
             }
         }
 
         let succ = self.propagate_through_expr(&body.value, self.exit_ln);
 
-        if self.closure_captures.is_none() {
+        if self.closure_min_captures.is_none() {
             // Either not a closure, or closure without any captured variables.
             // No need to determine liveness of captured variables, since there
             // are none.
@@ -1067,7 +1067,6 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 for (op, _op_sp) in asm.operands.iter().rev() {
                     match op {
                         hir::InlineAsmOperand::In { expr, .. }
-                        | hir::InlineAsmOperand::Const { expr, .. }
                         | hir::InlineAsmOperand::Sym { expr, .. } => {
                             succ = self.propagate_through_expr(expr, succ)
                         }
@@ -1085,6 +1084,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                             }
                             succ = self.propagate_through_expr(in_expr, succ);
                         }
+                        hir::InlineAsmOperand::Const { .. } => {}
                     }
                 }
                 succ
@@ -1094,7 +1094,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 let ia = &asm.inner;
                 let outputs = asm.outputs_exprs;
                 let inputs = asm.inputs_exprs;
-                let succ = ia.outputs.iter().zip(outputs).rev().fold(succ, |succ, (o, output)| {
+                let succ = iter::zip(&ia.outputs, outputs).rev().fold(succ, |succ, (o, output)| {
                     // see comment on places
                     // in propagate_through_place_components()
                     if o.is_indirect {
@@ -1221,7 +1221,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
         match path.res {
             Res::Local(hid) => {
                 let in_upvars = self.upvars.map_or(false, |u| u.contains_key(&hid));
-                let in_captures = self.closure_captures.map_or(false, |c| c.contains_key(&hid));
+                let in_captures = self.closure_min_captures.map_or(false, |c| c.contains_key(&hid));
 
                 match (in_upvars, in_captures) {
                     (false, _) | (true, true) => self.access_var(hir_id, hid, succ, acc, path.span),
@@ -1345,7 +1345,7 @@ fn check_expr<'tcx>(this: &mut Liveness<'_, 'tcx>, expr: &'tcx Expr<'tcx>) {
             }
 
             // Output operands must be places
-            for (o, output) in asm.inner.outputs.iter().zip(asm.outputs_exprs) {
+            for (o, output) in iter::zip(&asm.inner.outputs, asm.outputs_exprs) {
                 if !o.is_indirect {
                     this.check_place(output);
                 }
@@ -1422,52 +1422,52 @@ impl<'tcx> Liveness<'_, 'tcx> {
     }
 
     fn warn_about_unused_upvars(&self, entry_ln: LiveNode) {
-        let closure_captures = match self.closure_captures {
+        let closure_min_captures = match self.closure_min_captures {
             None => return,
-            Some(closure_captures) => closure_captures,
+            Some(closure_min_captures) => closure_min_captures,
         };
 
-        // If closure_captures is Some(), upvars must be Some() too.
-        let upvars = self.upvars.unwrap();
-        for &var_hir_id in closure_captures.keys() {
-            let upvar = upvars[&var_hir_id];
-            let var = self.variable(var_hir_id, upvar.span);
-            let upvar_id = ty::UpvarId {
-                var_path: ty::UpvarPath { hir_id: var_hir_id },
-                closure_expr_id: self.body_owner,
-            };
-            match self.typeck_results.upvar_capture(upvar_id) {
-                ty::UpvarCapture::ByValue(_) => {}
-                ty::UpvarCapture::ByRef(..) => continue,
-            };
-            if self.used_on_entry(entry_ln, var) {
-                if !self.live_on_entry(entry_ln, var) {
+        // If closure_min_captures is Some(), upvars must be Some() too.
+        for (&var_hir_id, min_capture_list) in closure_min_captures {
+            for captured_place in min_capture_list {
+                match captured_place.info.capture_kind {
+                    ty::UpvarCapture::ByValue(_) => {}
+                    ty::UpvarCapture::ByRef(..) => continue,
+                };
+                let span = captured_place.get_capture_kind_span(self.ir.tcx);
+                let var = self.variable(var_hir_id, span);
+                if self.used_on_entry(entry_ln, var) {
+                    if !self.live_on_entry(entry_ln, var) {
+                        if let Some(name) = self.should_warn(var) {
+                            self.ir.tcx.struct_span_lint_hir(
+                                lint::builtin::UNUSED_ASSIGNMENTS,
+                                var_hir_id,
+                                vec![span],
+                                |lint| {
+                                    lint.build(&format!(
+                                        "value captured by `{}` is never read",
+                                        name
+                                    ))
+                                    .help("did you mean to capture by reference instead?")
+                                    .emit();
+                                },
+                            );
+                        }
+                    }
+                } else {
                     if let Some(name) = self.should_warn(var) {
                         self.ir.tcx.struct_span_lint_hir(
-                            lint::builtin::UNUSED_ASSIGNMENTS,
+                            lint::builtin::UNUSED_VARIABLES,
                             var_hir_id,
-                            vec![upvar.span],
+                            vec![span],
                             |lint| {
-                                lint.build(&format!("value captured by `{}` is never read", name))
+                                lint.build(&format!("unused variable: `{}`", name))
                                     .help("did you mean to capture by reference instead?")
                                     .emit();
                             },
                         );
                     }
                 }
-            } else {
-                if let Some(name) = self.should_warn(var) {
-                    self.ir.tcx.struct_span_lint_hir(
-                        lint::builtin::UNUSED_VARIABLES,
-                        var_hir_id,
-                        vec![upvar.span],
-                        |lint| {
-                            lint.build(&format!("unused variable: `{}`", name))
-                                .help("did you mean to capture by reference instead?")
-                                .emit();
-                        },
-                    );
-                }
             }
         }
     }
diff --git a/compiler/rustc_passes/src/liveness/rwu_table.rs b/compiler/rustc_passes/src/liveness/rwu_table.rs
index a1a6f27398e..6d5983f53dc 100644
--- a/compiler/rustc_passes/src/liveness/rwu_table.rs
+++ b/compiler/rustc_passes/src/liveness/rwu_table.rs
@@ -1,4 +1,5 @@
 use crate::liveness::{LiveNode, Variable};
+use std::iter;
 
 #[derive(Clone, Copy)]
 pub(super) struct RWU {
@@ -91,7 +92,7 @@ impl RWUTable {
 
         let mut changed = false;
         let (dst_row, src_row) = self.pick2_rows_mut(dst, src);
-        for (dst_word, src_word) in dst_row.iter_mut().zip(src_row.iter()) {
+        for (dst_word, src_word) in iter::zip(dst_row, &*src_row) {
             let old = *dst_word;
             let new = *dst_word | src_word;
             *dst_word = new;
diff --git a/compiler/rustc_passes/src/naked_functions.rs b/compiler/rustc_passes/src/naked_functions.rs
index 93fb23c018b..89bc2e1a987 100644
--- a/compiler/rustc_passes/src/naked_functions.rs
+++ b/compiler/rustc_passes/src/naked_functions.rs
@@ -46,7 +46,7 @@ impl<'tcx> Visitor<'tcx> for CheckNakedFunctions<'tcx> {
         let fn_header;
 
         match fk {
-            FnKind::Closure(..) => {
+            FnKind::Closure => {
                 // Closures with a naked attribute are rejected during attribute
                 // check. Don't validate them any further.
                 return;
@@ -62,7 +62,8 @@ impl<'tcx> Visitor<'tcx> for CheckNakedFunctions<'tcx> {
             }
         }
 
-        let naked = fk.attrs().iter().any(|attr| attr.has_name(sym::naked));
+        let attrs = self.tcx.hir().attrs(hir_id);
+        let naked = attrs.iter().any(|attr| attr.has_name(sym::naked));
         if naked {
             let body = self.tcx.hir().body(body_id);
             check_abi(self.tcx, hir_id, fn_header.abi, ident_span);
diff --git a/compiler/rustc_passes/src/region.rs b/compiler/rustc_passes/src/region.rs
index b532021bed2..14a373c5942 100644
--- a/compiler/rustc_passes/src/region.rs
+++ b/compiler/rustc_passes/src/region.rs
@@ -23,14 +23,6 @@ use std::mem;
 
 #[derive(Debug, Copy, Clone)]
 pub struct Context {
-    /// The root of the current region tree. This is typically the id
-    /// of the innermost fn body. Each fn forms its own disjoint tree
-    /// in the region hierarchy. These fn bodies are themselves
-    /// arranged into a tree. See the "Modeling closures" section of
-    /// the README in `rustc_trait_selection::infer::region_constraints`
-    /// for more details.
-    root_id: Option<hir::ItemLocalId>,
-
     /// The scope that contains any new variables declared, plus its depth in
     /// the scope tree.
     var_parent: Option<(Scope, ScopeDepth)>,
@@ -743,11 +735,6 @@ impl<'tcx> Visitor<'tcx> for RegionResolutionVisitor<'tcx> {
         let outer_pessimistic_yield = mem::replace(&mut self.pessimistic_yield, false);
         self.terminating_scopes.insert(body.value.hir_id.local_id);
 
-        if let Some(root_id) = self.cx.root_id {
-            self.scope_tree.record_closure_parent(body.value.hir_id.local_id, root_id);
-        }
-        self.cx.root_id = Some(body.value.hir_id.local_id);
-
         self.enter_scope(Scope { id: body.value.hir_id.local_id, data: ScopeData::CallSite });
         self.enter_scope(Scope { id: body.value.hir_id.local_id, data: ScopeData::Arguments });
 
@@ -824,7 +811,7 @@ fn region_scope_tree(tcx: TyCtxt<'_>, def_id: DefId) -> &ScopeTree {
             tcx,
             scope_tree: ScopeTree::default(),
             expr_and_pat_count: 0,
-            cx: Context { root_id: None, parent: None, var_parent: None },
+            cx: Context { parent: None, var_parent: None },
             terminating_scopes: Default::default(),
             pessimistic_yield: false,
             fixup_scopes: vec![],
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index c4c10343743..9c4f9b1198c 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -9,7 +9,7 @@ use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
-use rustc_hir::{Generics, HirId, Item, StructField, TraitRef, Ty, TyKind, Variant};
+use rustc_hir::{FieldDef, Generics, HirId, Item, TraitRef, Ty, TyKind, Variant};
 use rustc_middle::hir::map::Map;
 use rustc_middle::middle::privacy::AccessLevels;
 use rustc_middle::middle::stability::{DeprecationEntry, Index};
@@ -22,6 +22,7 @@ use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 
 use std::cmp::Ordering;
+use std::iter;
 use std::mem::replace;
 use std::num::NonZeroU32;
 
@@ -97,7 +98,6 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
     fn annotate<F>(
         &mut self,
         hir_id: HirId,
-        attrs: &[Attribute],
         item_sp: Span,
         kind: AnnotationKind,
         inherit_deprecation: InheritDeprecation,
@@ -107,6 +107,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
     ) where
         F: FnOnce(&mut Self),
     {
+        let attrs = self.tcx.hir().attrs(hir_id);
         debug!("annotate(id = {:?}, attrs = {:?})", hir_id, attrs);
         let mut did_error = false;
         if !self.tcx.features().staged_api {
@@ -214,7 +215,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             {
                 // Explicit version of iter::order::lt to handle parse errors properly
                 for (dep_v, stab_v) in
-                    dep_since.as_str().split('.').zip(stab_since.as_str().split('.'))
+                    iter::zip(dep_since.as_str().split('.'), stab_since.as_str().split('.'))
                 {
                     match stab_v.parse::<u64>() {
                         Err(_) => {
@@ -385,7 +386,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
                 if let Some(ctor_hir_id) = sd.ctor_hir_id() {
                     self.annotate(
                         ctor_hir_id,
-                        &i.attrs,
                         i.span,
                         AnnotationKind::Required,
                         InheritDeprecation::Yes,
@@ -400,7 +400,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
 
         self.annotate(
             i.hir_id(),
-            &i.attrs,
             i.span,
             kind,
             InheritDeprecation::Yes,
@@ -414,7 +413,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
     fn visit_trait_item(&mut self, ti: &'tcx hir::TraitItem<'tcx>) {
         self.annotate(
             ti.hir_id(),
-            &ti.attrs,
             ti.span,
             AnnotationKind::Required,
             InheritDeprecation::Yes,
@@ -431,7 +429,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
             if self.in_trait_impl { AnnotationKind::Prohibited } else { AnnotationKind::Required };
         self.annotate(
             ii.hir_id(),
-            &ii.attrs,
             ii.span,
             kind,
             InheritDeprecation::Yes,
@@ -446,7 +443,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
     fn visit_variant(&mut self, var: &'tcx Variant<'tcx>, g: &'tcx Generics<'tcx>, item_id: HirId) {
         self.annotate(
             var.id,
-            &var.attrs,
             var.span,
             AnnotationKind::Required,
             InheritDeprecation::Yes,
@@ -456,7 +452,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
                 if let Some(ctor_hir_id) = var.data.ctor_hir_id() {
                     v.annotate(
                         ctor_hir_id,
-                        &var.attrs,
                         var.span,
                         AnnotationKind::Required,
                         InheritDeprecation::Yes,
@@ -471,17 +466,16 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
         )
     }
 
-    fn visit_struct_field(&mut self, s: &'tcx StructField<'tcx>) {
+    fn visit_field_def(&mut self, s: &'tcx FieldDef<'tcx>) {
         self.annotate(
             s.hir_id,
-            &s.attrs,
             s.span,
             AnnotationKind::Required,
             InheritDeprecation::Yes,
             InheritConstStability::No,
             InheritStability::Yes,
             |v| {
-                intravisit::walk_struct_field(v, s);
+                intravisit::walk_field_def(v, s);
             },
         );
     }
@@ -489,7 +483,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
     fn visit_foreign_item(&mut self, i: &'tcx hir::ForeignItem<'tcx>) {
         self.annotate(
             i.hir_id(),
-            &i.attrs,
             i.span,
             AnnotationKind::Required,
             InheritDeprecation::Yes,
@@ -504,7 +497,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
     fn visit_macro_def(&mut self, md: &'tcx hir::MacroDef<'tcx>) {
         self.annotate(
             md.hir_id(),
-            &md.attrs,
             md.span,
             AnnotationKind::Required,
             InheritDeprecation::Yes,
@@ -516,16 +508,14 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
 
     fn visit_generic_param(&mut self, p: &'tcx hir::GenericParam<'tcx>) {
         let kind = match &p.kind {
-            // FIXME(const_generics_defaults)
-            hir::GenericParamKind::Type { default, .. } if default.is_some() => {
-                AnnotationKind::Container
-            }
+            // Allow stability attributes on default generic arguments.
+            hir::GenericParamKind::Type { default: Some(_), .. }
+            | hir::GenericParamKind::Const { default: Some(_), .. } => AnnotationKind::Container,
             _ => AnnotationKind::Prohibited,
         };
 
         self.annotate(
             p.hir_id,
-            &p.attrs,
             p.span,
             kind,
             InheritDeprecation::No,
@@ -620,9 +610,9 @@ impl<'tcx> Visitor<'tcx> for MissingStabilityAnnotations<'tcx> {
         intravisit::walk_variant(self, var, g, item_id);
     }
 
-    fn visit_struct_field(&mut self, s: &'tcx StructField<'tcx>) {
+    fn visit_field_def(&mut self, s: &'tcx FieldDef<'tcx>) {
         self.check_missing_stability(s.hir_id, s.span);
-        intravisit::walk_struct_field(self, s);
+        intravisit::walk_field_def(self, s);
     }
 
     fn visit_foreign_item(&mut self, i: &'tcx hir::ForeignItem<'tcx>) {
@@ -696,8 +686,7 @@ fn new_index(tcx: TyCtxt<'tcx>) -> Index<'tcx> {
 
         annotator.annotate(
             hir::CRATE_HIR_ID,
-            &krate.item.attrs,
-            krate.item.span,
+            krate.item.inner,
             AnnotationKind::Required,
             InheritDeprecation::Yes,
             InheritConstStability::No,
@@ -762,8 +751,9 @@ impl Visitor<'tcx> for Checker<'tcx> {
                     // error if all involved types and traits are stable, because
                     // it will have no effect.
                     // See: https://github.com/rust-lang/rust/issues/55436
+                    let attrs = self.tcx.hir().attrs(item.hir_id());
                     if let (Some((Stability { level: attr::Unstable { .. }, .. }, span)), _) =
-                        attr::find_stability(&self.tcx.sess, &item.attrs, item.span)
+                        attr::find_stability(&self.tcx.sess, attrs, item.span)
                     {
                         let mut c = CheckTraitImplStable { tcx: self.tcx, fully_stable: true };
                         c.visit_ty(self_ty);
@@ -895,7 +885,7 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
     if tcx.stability().staged_api[&LOCAL_CRATE] {
         let krate = tcx.hir().krate();
         let mut missing = MissingStabilityAnnotations { tcx, access_levels };
-        missing.check_missing_stability(hir::CRATE_HIR_ID, krate.item.span);
+        missing.check_missing_stability(hir::CRATE_HIR_ID, krate.item.inner);
         intravisit::walk_crate(&mut missing, krate);
         krate.visit_all_item_likes(&mut missing.as_deep_visitor());
     }
diff --git a/compiler/rustc_passes/src/weak_lang_items.rs b/compiler/rustc_passes/src/weak_lang_items.rs
index daff94cb6d3..de369ba9bbb 100644
--- a/compiler/rustc_passes/src/weak_lang_items.rs
+++ b/compiler/rustc_passes/src/weak_lang_items.rs
@@ -97,7 +97,8 @@ impl<'a, 'tcx, 'v> Visitor<'v> for Context<'a, 'tcx> {
 
     fn visit_foreign_item(&mut self, i: &hir::ForeignItem<'_>) {
         let check_name = |attr, sym| self.tcx.sess.check_name(attr, sym);
-        if let Some((lang_item, _)) = lang_items::extract(check_name, &i.attrs) {
+        let attrs = self.tcx.hir().attrs(i.hir_id());
+        if let Some((lang_item, _)) = lang_items::extract(check_name, attrs) {
             self.register(lang_item, i.span);
         }
         intravisit::walk_foreign_item(self, i)
diff --git a/compiler/rustc_plugin_impl/src/build.rs b/compiler/rustc_plugin_impl/src/build.rs
index d5c287fb3bc..a49afa35e46 100644
--- a/compiler/rustc_plugin_impl/src/build.rs
+++ b/compiler/rustc_plugin_impl/src/build.rs
@@ -16,7 +16,8 @@ struct RegistrarFinder<'tcx> {
 impl<'v, 'tcx> ItemLikeVisitor<'v> for RegistrarFinder<'tcx> {
     fn visit_item(&mut self, item: &hir::Item<'_>) {
         if let hir::ItemKind::Fn(..) = item.kind {
-            if self.tcx.sess.contains_name(&item.attrs, sym::plugin_registrar) {
+            let attrs = self.tcx.hir().attrs(item.hir_id());
+            if self.tcx.sess.contains_name(attrs, sym::plugin_registrar) {
                 self.registrars.push((item.def_id, item.span));
             }
         }
diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs
index 6e0e1c5eeef..d37a5be2fe5 100644
--- a/compiler/rustc_privacy/src/lib.rs
+++ b/compiler/rustc_privacy/src/lib.rs
@@ -1,7 +1,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(in_band_lifetimes)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(control_flow_enum)]
 #![feature(try_blocks)]
 #![feature(associated_type_defaults)]
@@ -881,7 +881,8 @@ impl Visitor<'tcx> for EmbargoVisitor<'tcx> {
 
     fn visit_macro_def(&mut self, md: &'tcx hir::MacroDef<'tcx>) {
         // Non-opaque macros cannot make other items more accessible than they already are.
-        if attr::find_transparency(&self.tcx.sess, &md.attrs, md.ast.macro_rules).0
+        let attrs = self.tcx.hir().attrs(md.hir_id());
+        if attr::find_transparency(&self.tcx.sess, &attrs, md.ast.macro_rules).0
             != Transparency::Opaque
         {
             // `#[macro_export]`-ed `macro_rules!` are `Public` since they
@@ -927,8 +928,11 @@ impl ReachEverythingInTheInterfaceVisitor<'_, 'tcx> {
                         self.visit(self.ev.tcx.type_of(param.def_id));
                     }
                 }
-                GenericParamDefKind::Const => {
+                GenericParamDefKind::Const { has_default, .. } => {
                     self.visit(self.ev.tcx.type_of(param.def_id));
+                    if has_default {
+                        self.visit(self.ev.tcx.const_param_default(param.def_id));
+                    }
                 }
             }
         }
@@ -1110,7 +1114,7 @@ impl<'tcx> Visitor<'tcx> for NamePrivacyVisitor<'tcx> {
 
 ////////////////////////////////////////////////////////////////////////////////////////////
 /// Type privacy visitor, checks types for privacy and reports violations.
-/// Both explicitly written types and inferred types of expressions and patters are checked.
+/// Both explicitly written types and inferred types of expressions and patterns are checked.
 /// Checks are performed on "semantic" types regardless of names and their hygiene.
 ////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -1697,9 +1701,9 @@ impl<'a, 'tcx> Visitor<'tcx> for ObsoleteVisiblePrivateTypesVisitor<'a, 'tcx> {
         }
     }
 
-    fn visit_struct_field(&mut self, s: &'tcx hir::StructField<'tcx>) {
+    fn visit_field_def(&mut self, s: &'tcx hir::FieldDef<'tcx>) {
         if s.vis.node.is_pub() || self.in_variant {
-            intravisit::walk_struct_field(self, s);
+            intravisit::walk_field_def(self, s);
         }
     }
 
@@ -1740,7 +1744,8 @@ impl SearchInterfaceForPrivateItemsVisitor<'tcx> {
                         self.visit(self.tcx.type_of(param.def_id));
                     }
                 }
-                GenericParamDefKind::Const => {
+                // FIXME(const_evaluatable_checked): May want to look inside const here
+                GenericParamDefKind::Const { .. } => {
                     self.visit(self.tcx.type_of(param.def_id));
                 }
             }
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index c88b766a55a..383e30ca29f 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -9,7 +9,7 @@ doctest = false
 
 [dependencies]
 measureme = "9.0.0"
-rustc-rayon-core = "0.3.0"
+rustc-rayon-core = "0.3.1"
 tracing = "0.1"
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
diff --git a/compiler/rustc_query_impl/src/keys.rs b/compiler/rustc_query_impl/src/keys.rs
index 1ae5bf12cab..1fdb37398f9 100644
--- a/compiler/rustc_query_impl/src/keys.rs
+++ b/compiler/rustc_query_impl/src/keys.rs
@@ -228,6 +228,15 @@ impl<'tcx> Key for (&'tcx ty::Const<'tcx>, mir::Field) {
     }
 }
 
+impl<'tcx> Key for mir::interpret::ConstAlloc<'tcx> {
+    fn query_crate(&self) -> CrateNum {
+        LOCAL_CRATE
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
 impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
     fn query_crate(&self) -> CrateNum {
         self.def_id().krate
@@ -246,6 +255,15 @@ impl<'tcx> Key for GenericArg<'tcx> {
     }
 }
 
+impl<'tcx> Key for mir::ConstantKind<'tcx> {
+    fn query_crate(&self) -> CrateNum {
+        LOCAL_CRATE
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
 impl<'tcx> Key for &'tcx ty::Const<'tcx> {
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index e9314797fbd..00d886000fa 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -19,8 +19,7 @@ extern crate tracing;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_errors::{DiagnosticBuilder, Handler};
-use rustc_hir::def_id::CrateNum;
-use rustc_index::vec::IndexVec;
+use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::dep_graph;
 use rustc_middle::ich::StableHashingContext;
 use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 37a176de941..ee914fa1ba9 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -390,13 +390,12 @@ macro_rules! define_queries {
 
             #[inline]
             fn compute(tcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
-                let provider = tcx.queries.providers.get(key.query_crate())
-                    // HACK(eddyb) it's possible crates may be loaded after
-                    // the query engine is created, and because crate loading
-                    // is not yet integrated with the query engine, such crates
-                    // would be missing appropriate entries in `providers`.
-                    .unwrap_or(&tcx.queries.fallback_extern_providers)
-                    .$name;
+                let is_local = key.query_crate() == LOCAL_CRATE;
+                let provider = if is_local {
+                    tcx.queries.local_providers.$name
+                } else {
+                    tcx.queries.extern_providers.$name
+                };
                 provider(*tcx, key)
             }
 
@@ -439,6 +438,11 @@ macro_rules! define_queries {
                 try_load_from_on_disk_cache: |_, _| {},
             };
 
+            pub const CompileMonoItem: QueryStruct = QueryStruct {
+                force_from_dep_node: |_, _| false,
+                try_load_from_on_disk_cache: |_, _| {},
+            };
+
             $(pub const $name: QueryStruct = {
                 const is_anon: bool = is_anon!([$($modifiers)*]);
 
@@ -478,10 +482,7 @@ macro_rules! define_queries {
                         return
                     }
 
-                    debug_assert!(tcx.dep_graph
-                                     .node_color(dep_node)
-                                     .map(|c| c.is_green())
-                                     .unwrap_or(false));
+                    debug_assert!(tcx.dep_graph.is_green(dep_node));
 
                     let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
                     if queries::$name::cache_on_disk(tcx, &key, None) {
@@ -507,8 +508,8 @@ macro_rules! define_queries_struct {
     (tcx: $tcx:tt,
      input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
         pub struct Queries<$tcx> {
-            providers: IndexVec<CrateNum, Providers>,
-            fallback_extern_providers: Box<Providers>,
+            local_providers: Box<Providers>,
+            extern_providers: Box<Providers>,
 
             $($(#[$attr])*  $name: QueryState<
                 crate::dep_graph::DepKind,
@@ -518,12 +519,12 @@ macro_rules! define_queries_struct {
 
         impl<$tcx> Queries<$tcx> {
             pub fn new(
-                providers: IndexVec<CrateNum, Providers>,
-                fallback_extern_providers: Providers,
+                local_providers: Providers,
+                extern_providers: Providers,
             ) -> Self {
                 Queries {
-                    providers,
-                    fallback_extern_providers: Box::new(fallback_extern_providers),
+                    local_providers: Box::new(local_providers),
+                    extern_providers: Box::new(extern_providers),
                     $($name: Default::default()),*
                 }
             }
diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml
index 7d3357f8fa2..19512dc1f64 100644
--- a/compiler/rustc_query_system/Cargo.toml
+++ b/compiler/rustc_query_system/Cargo.toml
@@ -10,7 +10,7 @@ doctest = false
 [dependencies]
 rustc_arena = { path = "../rustc_arena" }
 tracing = "0.1"
-rustc-rayon-core = "0.3.0"
+rustc-rayon-core = "0.3.1"
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_macros = { path = "../rustc_macros" }
diff --git a/compiler/rustc_query_system/src/cache.rs b/compiler/rustc_query_system/src/cache.rs
index c6dc7b4fe28..d592812f79b 100644
--- a/compiler/rustc_query_system/src/cache.rs
+++ b/compiler/rustc_query_system/src/cache.rs
@@ -3,7 +3,6 @@
 use crate::dep_graph::{DepContext, DepNodeIndex};
 
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::sync::HashMapExt;
 use rustc_data_structures::sync::Lock;
 
 use std::hash::Hash;
@@ -34,13 +33,6 @@ impl<Key: Eq + Hash, Value: Clone> Cache<Key, Value> {
     pub fn insert(&self, key: Key, dep_node: DepNodeIndex, value: Value) {
         self.hashmap.borrow_mut().insert(key, WithDepNode::new(dep_node, value));
     }
-
-    pub fn insert_same(&self, key: Key, dep_node: DepNodeIndex, value: Value)
-    where
-        Value: Eq,
-    {
-        self.hashmap.borrow_mut().insert_same(key, WithDepNode::new(dep_node, value));
-    }
 }
 
 #[derive(Clone, Eq, PartialEq)]
diff --git a/compiler/rustc_query_system/src/dep_graph/debug.rs b/compiler/rustc_query_system/src/dep_graph/debug.rs
index 718a2f1039a..a544ac2c343 100644
--- a/compiler/rustc_query_system/src/dep_graph/debug.rs
+++ b/compiler/rustc_query_system/src/dep_graph/debug.rs
@@ -1,6 +1,8 @@
 //! Code for debugging the dep-graph.
 
-use super::{DepKind, DepNode};
+use super::{DepKind, DepNode, DepNodeIndex};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Lock;
 use std::error::Error;
 
 /// A dep-node filter goes from a user-defined string to a query over
@@ -34,13 +36,14 @@ impl DepNodeFilter {
 
 /// A filter like `F -> G` where `F` and `G` are valid dep-node
 /// filters. This can be used to test the source/target independently.
-pub struct EdgeFilter {
+pub struct EdgeFilter<K: DepKind> {
     pub source: DepNodeFilter,
     pub target: DepNodeFilter,
+    pub index_to_node: Lock<FxHashMap<DepNodeIndex, DepNode<K>>>,
 }
 
-impl EdgeFilter {
-    pub fn new(test: &str) -> Result<EdgeFilter, Box<dyn Error>> {
+impl<K: DepKind> EdgeFilter<K> {
+    pub fn new(test: &str) -> Result<EdgeFilter<K>, Box<dyn Error>> {
         let parts: Vec<_> = test.split("->").collect();
         if parts.len() != 2 {
             Err(format!("expected a filter like `a&b -> c&d`, not `{}`", test).into())
@@ -48,11 +51,13 @@ impl EdgeFilter {
             Ok(EdgeFilter {
                 source: DepNodeFilter::new(parts[0]),
                 target: DepNodeFilter::new(parts[1]),
+                index_to_node: Lock::new(FxHashMap::default()),
             })
         }
     }
 
-    pub fn test<K: DepKind>(&self, source: &DepNode<K>, target: &DepNode<K>) -> bool {
+    #[cfg(debug_assertions)]
+    pub fn test(&self, source: &DepNode<K>, target: &DepNode<K>) -> bool {
         self.source.test(source) && self.target.test(target)
     }
 }
diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs
index f55e2f777a2..59ef6052a60 100644
--- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs
@@ -26,7 +26,7 @@
 //!   could not be instantiated because the current compilation session
 //!   contained no `DefId` for thing that had been removed.
 //!
-//! `DepNode` definition happens in `librustc_middle` with the `define_dep_nodes!()` macro.
+//! `DepNode` definition happens in `rustc_middle` with the `define_dep_nodes!()` macro.
 //! This macro defines the `DepKind` enum and a corresponding `DepConstructor` enum. The
 //! `DepConstructor` enum links a `DepKind` to the parameters that are needed at runtime in order
 //! to construct a valid `DepNode` fingerprint.
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 0f25572170f..7a0fc320663 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -1,31 +1,33 @@
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
+use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{self, Sharded};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, LockGuard, Lrc, Ordering};
+use rustc_data_structures::steal::Steal;
+use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering};
 use rustc_data_structures::unlikely;
 use rustc_errors::Diagnostic;
-use rustc_index::vec::{Idx, IndexVec};
-use rustc_serialize::{Encodable, Encoder};
+use rustc_index::vec::IndexVec;
+use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 
 use parking_lot::{Condvar, Mutex};
 use smallvec::{smallvec, SmallVec};
 use std::collections::hash_map::Entry;
-use std::env;
 use std::hash::Hash;
 use std::marker::PhantomData;
 use std::mem;
-use std::ops::Range;
 use std::sync::atomic::Ordering::Relaxed;
 
-use super::debug::EdgeFilter;
 use super::prev::PreviousDepGraph;
 use super::query::DepGraphQuery;
-use super::serialized::SerializedDepNodeIndex;
+use super::serialized::{GraphEncoder, SerializedDepNodeIndex};
 use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId};
 use crate::query::QueryContext;
 
+#[cfg(debug_assertions)]
+use {super::debug::EdgeFilter, std::env};
+
 #[derive(Clone)]
 pub struct DepGraph<K: DepKind> {
     data: Option<Lrc<DepGraphData<K>>>,
@@ -109,6 +111,9 @@ impl<K: DepKind> DepGraph<K> {
     pub fn new(
         prev_graph: PreviousDepGraph<K>,
         prev_work_products: FxHashMap<WorkProductId, WorkProduct>,
+        encoder: FileEncoder,
+        record_graph: bool,
+        record_stats: bool,
     ) -> DepGraph<K> {
         let prev_graph_node_count = prev_graph.node_count();
 
@@ -116,7 +121,12 @@ impl<K: DepKind> DepGraph<K> {
             data: Some(Lrc::new(DepGraphData {
                 previous_work_products: prev_work_products,
                 dep_node_debug: Default::default(),
-                current: CurrentDepGraph::new(prev_graph_node_count),
+                current: CurrentDepGraph::new(
+                    prev_graph_node_count,
+                    encoder,
+                    record_graph,
+                    record_stats,
+                ),
                 emitting_diagnostics: Default::default(),
                 emitting_diagnostics_cond_var: Condvar::new(),
                 previous: prev_graph,
@@ -136,62 +146,10 @@ impl<K: DepKind> DepGraph<K> {
         self.data.is_some()
     }
 
-    pub fn query(&self) -> DepGraphQuery<K> {
-        let data = self.data.as_ref().unwrap();
-        let previous = &data.previous;
-
-        // Note locking order: `prev_index_to_index`, then `data`.
-        let prev_index_to_index = data.current.prev_index_to_index.lock();
-        let data = data.current.data.lock();
-        let node_count = data.hybrid_indices.len();
-        let edge_count = self.edge_count(&data);
-
-        let mut nodes = Vec::with_capacity(node_count);
-        let mut edge_list_indices = Vec::with_capacity(node_count);
-        let mut edge_list_data = Vec::with_capacity(edge_count);
-
-        // See `DepGraph`'s `Encodable` implementation for notes on the approach used here.
-
-        edge_list_data.extend(data.unshared_edges.iter().map(|i| i.index()));
-
-        for &hybrid_index in data.hybrid_indices.iter() {
-            match hybrid_index.into() {
-                HybridIndex::New(new_index) => {
-                    nodes.push(data.new.nodes[new_index]);
-                    let edges = &data.new.edges[new_index];
-                    edge_list_indices.push((edges.start.index(), edges.end.index()));
-                }
-                HybridIndex::Red(red_index) => {
-                    nodes.push(previous.index_to_node(data.red.node_indices[red_index]));
-                    let edges = &data.red.edges[red_index];
-                    edge_list_indices.push((edges.start.index(), edges.end.index()));
-                }
-                HybridIndex::LightGreen(lg_index) => {
-                    nodes.push(previous.index_to_node(data.light_green.node_indices[lg_index]));
-                    let edges = &data.light_green.edges[lg_index];
-                    edge_list_indices.push((edges.start.index(), edges.end.index()));
-                }
-                HybridIndex::DarkGreen(prev_index) => {
-                    nodes.push(previous.index_to_node(prev_index));
-
-                    let edges_iter = previous
-                        .edge_targets_from(prev_index)
-                        .iter()
-                        .map(|&dst| prev_index_to_index[dst].unwrap().index());
-
-                    let start = edge_list_data.len();
-                    edge_list_data.extend(edges_iter);
-                    let end = edge_list_data.len();
-                    edge_list_indices.push((start, end));
-                }
-            }
+    pub fn with_query(&self, f: impl Fn(&DepGraphQuery<K>)) {
+        if let Some(data) = &self.data {
+            data.current.encoder.borrow().with_query(f)
         }
-
-        debug_assert_eq!(nodes.len(), node_count);
-        debug_assert_eq!(edge_list_indices.len(), node_count);
-        debug_assert_eq!(edge_list_data.len(), edge_count);
-
-        DepGraphQuery::new(&nodes[..], &edge_list_indices[..], &edge_list_data[..])
     }
 
     pub fn assert_ignored(&self) {
@@ -283,56 +241,16 @@ impl<K: DepKind> DepGraph<K> {
             let print_status = cfg!(debug_assertions) && dcx.sess().opts.debugging_opts.dep_tasks;
 
             // Intern the new `DepNode`.
-            let dep_node_index = if let Some(prev_index) = data.previous.node_to_index_opt(&key) {
-                // Determine the color and index of the new `DepNode`.
-                let (color, dep_node_index) = if let Some(current_fingerprint) = current_fingerprint
-                {
-                    if current_fingerprint == data.previous.fingerprint_by_index(prev_index) {
-                        if print_status {
-                            eprintln!("[task::green] {:?}", key);
-                        }
-
-                        // This is a light green node: it existed in the previous compilation,
-                        // its query was re-executed, and it has the same result as before.
-                        let dep_node_index =
-                            data.current.intern_light_green_node(&data.previous, prev_index, edges);
-
-                        (DepNodeColor::Green(dep_node_index), dep_node_index)
-                    } else {
-                        if print_status {
-                            eprintln!("[task::red] {:?}", key);
-                        }
-
-                        // This is a red node: it existed in the previous compilation, its query
-                        // was re-executed, but it has a different result from before.
-                        let dep_node_index = data.current.intern_red_node(
-                            &data.previous,
-                            prev_index,
-                            edges,
-                            current_fingerprint,
-                        );
-
-                        (DepNodeColor::Red, dep_node_index)
-                    }
-                } else {
-                    if print_status {
-                        eprintln!("[task::unknown] {:?}", key);
-                    }
-
-                    // This is a red node, effectively: it existed in the previous compilation
-                    // session, its query was re-executed, but it doesn't compute a result hash
-                    // (i.e. it represents a `no_hash` query), so we have no way of determining
-                    // whether or not the result was the same as before.
-                    let dep_node_index = data.current.intern_red_node(
-                        &data.previous,
-                        prev_index,
-                        edges,
-                        Fingerprint::ZERO,
-                    );
-
-                    (DepNodeColor::Red, dep_node_index)
-                };
+            let (dep_node_index, prev_and_color) = data.current.intern_node(
+                dcx.profiler(),
+                &data.previous,
+                key,
+                edges,
+                current_fingerprint,
+                print_status,
+            );
 
+            if let Some((prev_index, color)) = prev_and_color {
                 debug_assert!(
                     data.colors.get(prev_index).is_none(),
                     "DepGraph::with_task() - Duplicate DepNodeColor \
@@ -341,20 +259,7 @@ impl<K: DepKind> DepGraph<K> {
                 );
 
                 data.colors.insert(prev_index, color);
-                dep_node_index
-            } else {
-                if print_status {
-                    eprintln!("[task::new] {:?}", key);
-                }
-
-                // This is a new node: it didn't exist in the previous compilation session.
-                data.current.intern_new_node(
-                    &data.previous,
-                    key,
-                    edges,
-                    current_fingerprint.unwrap_or(Fingerprint::ZERO),
-                )
-            };
+            }
 
             (result, dep_node_index)
         } else {
@@ -368,7 +273,12 @@ impl<K: DepKind> DepGraph<K> {
 
     /// Executes something within an "anonymous" task, that is, a task the
     /// `DepNode` of which is determined by the list of inputs it read from.
-    pub fn with_anon_task<OP, R>(&self, dep_kind: K, op: OP) -> (R, DepNodeIndex)
+    pub fn with_anon_task<Ctxt: DepContext<DepKind = K>, OP, R>(
+        &self,
+        cx: Ctxt,
+        dep_kind: K,
+        op: OP,
+    ) -> (R, DepNodeIndex)
     where
         OP: FnOnce() -> R,
     {
@@ -396,7 +306,7 @@ impl<K: DepKind> DepGraph<K> {
             };
 
             let dep_node_index = data.current.intern_new_node(
-                &data.previous,
+                cx.profiler(),
                 target_dep_node,
                 task_deps.reads,
                 Fingerprint::ZERO,
@@ -451,7 +361,7 @@ impl<K: DepKind> DepGraph<K> {
                         {
                             if let Some(target) = task_deps.node {
                                 if let Some(ref forbidden_edge) = data.current.forbidden_edge {
-                                    let src = self.dep_node_of(dep_node_index);
+                                    let src = forbidden_edge.index_to_node.lock()[&dep_node_index];
                                     if forbidden_edge.test(&src, &target) {
                                         panic!("forbidden edge {:?} -> {:?} created", src, target)
                                     }
@@ -488,38 +398,6 @@ impl<K: DepKind> DepGraph<K> {
         self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some()
     }
 
-    #[inline]
-    pub fn dep_node_of(&self, dep_node_index: DepNodeIndex) -> DepNode<K> {
-        let data = self.data.as_ref().unwrap();
-        let previous = &data.previous;
-        let data = data.current.data.lock();
-
-        match data.hybrid_indices[dep_node_index].into() {
-            HybridIndex::New(new_index) => data.new.nodes[new_index],
-            HybridIndex::Red(red_index) => previous.index_to_node(data.red.node_indices[red_index]),
-            HybridIndex::LightGreen(light_green_index) => {
-                previous.index_to_node(data.light_green.node_indices[light_green_index])
-            }
-            HybridIndex::DarkGreen(prev_index) => previous.index_to_node(prev_index),
-        }
-    }
-
-    #[inline]
-    pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
-        let data = self.data.as_ref().unwrap();
-        let previous = &data.previous;
-        let data = data.current.data.lock();
-
-        match data.hybrid_indices[dep_node_index].into() {
-            HybridIndex::New(new_index) => data.new.fingerprints[new_index],
-            HybridIndex::Red(red_index) => data.red.fingerprints[red_index],
-            HybridIndex::LightGreen(light_green_index) => {
-                previous.fingerprint_by_index(data.light_green.node_indices[light_green_index])
-            }
-            HybridIndex::DarkGreen(prev_index) => previous.fingerprint_by_index(prev_index),
-        }
-    }
-
     pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
         self.data.as_ref().unwrap().previous.fingerprint_of(dep_node)
     }
@@ -554,29 +432,13 @@ impl<K: DepKind> DepGraph<K> {
         self.data.as_ref()?.dep_node_debug.borrow().get(&dep_node).cloned()
     }
 
-    fn edge_count(&self, node_data: &LockGuard<'_, DepNodeData<K>>) -> usize {
-        let data = self.data.as_ref().unwrap();
-        let previous = &data.previous;
-
-        let mut edge_count = node_data.unshared_edges.len();
-
-        for &hybrid_index in node_data.hybrid_indices.iter() {
-            if let HybridIndex::DarkGreen(prev_index) = hybrid_index.into() {
-                edge_count += previous.edge_targets_from(prev_index).len()
-            }
-        }
-
-        edge_count
-    }
-
-    pub fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
+    fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
         if let Some(ref data) = self.data {
             if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) {
                 return data.colors.get(prev_index);
             } else {
-                // This is a node that did not exist in the previous compilation
-                // session, so we consider it to be red.
-                return Some(DepNodeColor::Red);
+                // This is a node that did not exist in the previous compilation session.
+                return None;
             }
         }
 
@@ -775,11 +637,13 @@ impl<K: DepKind> DepGraph<K> {
 
         // There may be multiple threads trying to mark the same dep node green concurrently
 
-        let dep_node_index = {
-            // We allocating an entry for the node in the current dependency graph and
-            // adding all the appropriate edges imported from the previous graph
-            data.current.intern_dark_green_node(&data.previous, prev_dep_node_index)
-        };
+        // We allocating an entry for the node in the current dependency graph and
+        // adding all the appropriate edges imported from the previous graph
+        let dep_node_index = data.current.promote_node_and_deps_to_current(
+            tcx.dep_context().profiler(),
+            &data.previous,
+            prev_dep_node_index,
+        );
 
         // ... emitting any stored diagnostic ...
 
@@ -862,6 +726,12 @@ impl<K: DepKind> DepGraph<K> {
         }
     }
 
+    // Returns true if the given node has been marked as red during the
+    // current compilation session. Used in various assertions
+    pub fn is_red(&self, dep_node: &DepNode<K>) -> bool {
+        self.node_color(dep_node) == Some(DepNodeColor::Red)
+    }
+
     // Returns true if the given node has been marked as green during the
     // current compilation session. Used in various assertions
     pub fn is_green(&self, dep_node: &DepNode<K>) -> bool {
@@ -911,106 +781,20 @@ impl<K: DepKind> DepGraph<K> {
     }
 
     pub fn print_incremental_info(&self) {
-        #[derive(Clone)]
-        struct Stat<Kind: DepKind> {
-            kind: Kind,
-            node_counter: u64,
-            edge_counter: u64,
+        if let Some(data) = &self.data {
+            data.current.encoder.borrow().print_incremental_info(
+                data.current.total_read_count.load(Relaxed),
+                data.current.total_duplicate_read_count.load(Relaxed),
+            )
         }
+    }
 
-        let data = self.data.as_ref().unwrap();
-        let prev = &data.previous;
-        let current = &data.current;
-        let data = current.data.lock();
-
-        let mut stats: FxHashMap<_, Stat<K>> = FxHashMap::with_hasher(Default::default());
-
-        for &hybrid_index in data.hybrid_indices.iter() {
-            let (kind, edge_count) = match hybrid_index.into() {
-                HybridIndex::New(new_index) => {
-                    let kind = data.new.nodes[new_index].kind;
-                    let edge_range = &data.new.edges[new_index];
-                    (kind, edge_range.end.as_usize() - edge_range.start.as_usize())
-                }
-                HybridIndex::Red(red_index) => {
-                    let kind = prev.index_to_node(data.red.node_indices[red_index]).kind;
-                    let edge_range = &data.red.edges[red_index];
-                    (kind, edge_range.end.as_usize() - edge_range.start.as_usize())
-                }
-                HybridIndex::LightGreen(lg_index) => {
-                    let kind = prev.index_to_node(data.light_green.node_indices[lg_index]).kind;
-                    let edge_range = &data.light_green.edges[lg_index];
-                    (kind, edge_range.end.as_usize() - edge_range.start.as_usize())
-                }
-                HybridIndex::DarkGreen(prev_index) => {
-                    let kind = prev.index_to_node(prev_index).kind;
-                    let edge_count = prev.edge_targets_from(prev_index).len();
-                    (kind, edge_count)
-                }
-            };
-
-            let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
-            stat.node_counter += 1;
-            stat.edge_counter += edge_count as u64;
-        }
-
-        let total_node_count = data.hybrid_indices.len();
-        let total_edge_count = self.edge_count(&data);
-
-        // Drop the lock guard.
-        std::mem::drop(data);
-
-        let mut stats: Vec<_> = stats.values().cloned().collect();
-        stats.sort_by_key(|s| -(s.node_counter as i64));
-
-        const SEPARATOR: &str = "[incremental] --------------------------------\
-                                 ----------------------------------------------\
-                                 ------------";
-
-        eprintln!("[incremental]");
-        eprintln!("[incremental] DepGraph Statistics");
-        eprintln!("{}", SEPARATOR);
-        eprintln!("[incremental]");
-        eprintln!("[incremental] Total Node Count: {}", total_node_count);
-        eprintln!("[incremental] Total Edge Count: {}", total_edge_count);
-
-        if cfg!(debug_assertions) {
-            let total_edge_reads = current.total_read_count.load(Relaxed);
-            let total_duplicate_edge_reads = current.total_duplicate_read_count.load(Relaxed);
-
-            eprintln!("[incremental] Total Edge Reads: {}", total_edge_reads);
-            eprintln!("[incremental] Total Duplicate Edge Reads: {}", total_duplicate_edge_reads);
-        }
-
-        eprintln!("[incremental]");
-
-        eprintln!(
-            "[incremental]  {:<36}| {:<17}| {:<12}| {:<17}|",
-            "Node Kind", "Node Frequency", "Node Count", "Avg. Edge Count"
-        );
-
-        eprintln!(
-            "[incremental] -------------------------------------\
-                  |------------------\
-                  |-------------\
-                  |------------------|"
-        );
-
-        for stat in stats {
-            let node_kind_ratio = (100.0 * (stat.node_counter as f64)) / (total_node_count as f64);
-            let node_kind_avg_edges = (stat.edge_counter as f64) / (stat.node_counter as f64);
-
-            eprintln!(
-                "[incremental]  {:<36}|{:>16.1}% |{:>12} |{:>17.1} |",
-                format!("{:?}", stat.kind),
-                node_kind_ratio,
-                stat.node_counter,
-                node_kind_avg_edges,
-            );
+    pub fn encode(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
+        if let Some(data) = &self.data {
+            data.current.encoder.steal().finish(profiler)
+        } else {
+            Ok(())
         }
-
-        eprintln!("{}", SEPARATOR);
-        eprintln!("[incremental]");
     }
 
     fn next_virtual_depnode_index(&self) -> DepNodeIndex {
@@ -1019,142 +803,6 @@ impl<K: DepKind> DepGraph<K> {
     }
 }
 
-impl<E: Encoder, K: DepKind + Encodable<E>> Encodable<E> for DepGraph<K> {
-    fn encode(&self, e: &mut E) -> Result<(), E::Error> {
-        // We used to serialize the dep graph by creating and serializing a `SerializedDepGraph`
-        // using data copied from the `DepGraph`. But copying created a large memory spike, so we
-        // now serialize directly from the `DepGraph` as if it's a `SerializedDepGraph`. Because we
-        // deserialize that data into a `SerializedDepGraph` in the next compilation session, we
-        // need `DepGraph`'s `Encodable` and `SerializedDepGraph`'s `Decodable` implementations to
-        // be in sync. If you update this encoding, be sure to update the decoding, and vice-versa.
-
-        let data = self.data.as_ref().unwrap();
-        let prev = &data.previous;
-
-        // Note locking order: `prev_index_to_index`, then `data`.
-        let prev_index_to_index = data.current.prev_index_to_index.lock();
-        let data = data.current.data.lock();
-        let new = &data.new;
-        let red = &data.red;
-        let lg = &data.light_green;
-
-        let node_count = data.hybrid_indices.len();
-        let edge_count = self.edge_count(&data);
-
-        // `rustc_middle::ty::query::OnDiskCache` expects nodes to be encoded in `DepNodeIndex`
-        // order. The edges in `edge_list_data` don't need to be in a particular order, as long as
-        // each node references its edges as a contiguous range within it. Therefore, we can encode
-        // `edge_list_data` directly from `unshared_edges`. It meets the above requirements, as
-        // each non-dark-green node already knows the range of edges to reference within it, which
-        // they'll encode in `edge_list_indices`. Dark green nodes, however, don't have their edges
-        // in `unshared_edges`, so need to add them to `edge_list_data`.
-
-        use HybridIndex::*;
-
-        // Encoded values (nodes, etc.) are explicitly typed below to avoid inadvertently
-        // serializing data in the wrong format (i.e. one incompatible with `SerializedDepGraph`).
-        e.emit_struct("SerializedDepGraph", 4, |e| {
-            e.emit_struct_field("nodes", 0, |e| {
-                // `SerializedDepGraph` expects this to be encoded as a sequence of `DepNode`s.
-                e.emit_seq(node_count, |e| {
-                    for (seq_index, &hybrid_index) in data.hybrid_indices.iter().enumerate() {
-                        let node: DepNode<K> = match hybrid_index.into() {
-                            New(i) => new.nodes[i],
-                            Red(i) => prev.index_to_node(red.node_indices[i]),
-                            LightGreen(i) => prev.index_to_node(lg.node_indices[i]),
-                            DarkGreen(prev_index) => prev.index_to_node(prev_index),
-                        };
-
-                        e.emit_seq_elt(seq_index, |e| node.encode(e))?;
-                    }
-
-                    Ok(())
-                })
-            })?;
-
-            e.emit_struct_field("fingerprints", 1, |e| {
-                // `SerializedDepGraph` expects this to be encoded as a sequence of `Fingerprints`s.
-                e.emit_seq(node_count, |e| {
-                    for (seq_index, &hybrid_index) in data.hybrid_indices.iter().enumerate() {
-                        let fingerprint: Fingerprint = match hybrid_index.into() {
-                            New(i) => new.fingerprints[i],
-                            Red(i) => red.fingerprints[i],
-                            LightGreen(i) => prev.fingerprint_by_index(lg.node_indices[i]),
-                            DarkGreen(prev_index) => prev.fingerprint_by_index(prev_index),
-                        };
-
-                        e.emit_seq_elt(seq_index, |e| fingerprint.encode(e))?;
-                    }
-
-                    Ok(())
-                })
-            })?;
-
-            e.emit_struct_field("edge_list_indices", 2, |e| {
-                // `SerializedDepGraph` expects this to be encoded as a sequence of `(u32, u32)`s.
-                e.emit_seq(node_count, |e| {
-                    // Dark green node edges start after the unshared (all other nodes') edges.
-                    let mut dark_green_edge_index = data.unshared_edges.len();
-
-                    for (seq_index, &hybrid_index) in data.hybrid_indices.iter().enumerate() {
-                        let edge_indices: (u32, u32) = match hybrid_index.into() {
-                            New(i) => (new.edges[i].start.as_u32(), new.edges[i].end.as_u32()),
-                            Red(i) => (red.edges[i].start.as_u32(), red.edges[i].end.as_u32()),
-                            LightGreen(i) => (lg.edges[i].start.as_u32(), lg.edges[i].end.as_u32()),
-                            DarkGreen(prev_index) => {
-                                let edge_count = prev.edge_targets_from(prev_index).len();
-                                let start = dark_green_edge_index as u32;
-                                dark_green_edge_index += edge_count;
-                                let end = dark_green_edge_index as u32;
-                                (start, end)
-                            }
-                        };
-
-                        e.emit_seq_elt(seq_index, |e| edge_indices.encode(e))?;
-                    }
-
-                    assert_eq!(dark_green_edge_index, edge_count);
-
-                    Ok(())
-                })
-            })?;
-
-            e.emit_struct_field("edge_list_data", 3, |e| {
-                // `SerializedDepGraph` expects this to be encoded as a sequence of
-                // `SerializedDepNodeIndex`.
-                e.emit_seq(edge_count, |e| {
-                    for (seq_index, &edge) in data.unshared_edges.iter().enumerate() {
-                        let serialized_edge = SerializedDepNodeIndex::new(edge.index());
-                        e.emit_seq_elt(seq_index, |e| serialized_edge.encode(e))?;
-                    }
-
-                    let mut seq_index = data.unshared_edges.len();
-
-                    for &hybrid_index in data.hybrid_indices.iter() {
-                        if let DarkGreen(prev_index) = hybrid_index.into() {
-                            for &edge in prev.edge_targets_from(prev_index) {
-                                // Dark green node edges are stored in the previous graph
-                                // and must be converted to edges in the current graph,
-                                // and then serialized as `SerializedDepNodeIndex`.
-                                let serialized_edge = SerializedDepNodeIndex::new(
-                                    prev_index_to_index[edge].as_ref().unwrap().index(),
-                                );
-
-                                e.emit_seq_elt(seq_index, |e| serialized_edge.encode(e))?;
-                                seq_index += 1;
-                            }
-                        }
-                    }
-
-                    assert_eq!(seq_index, edge_count);
-
-                    Ok(())
-                })
-            })
-        })
-    }
-}
-
 /// A "work product" is an intermediate result that we save into the
 /// incremental directory for later re-use. The primary example are
 /// the object files that we save for each partition at code
@@ -1193,216 +841,20 @@ pub struct WorkProduct {
     pub saved_file: Option<String>,
 }
 
-// The maximum value of the follow index types leaves the upper two bits unused
-// so that we can store multiple index types in `CompressedHybridIndex`, and use
-// those bits to encode which index type it contains.
-
-// Index type for `NewDepNodeData`.
-rustc_index::newtype_index! {
-    struct NewDepNodeIndex {
-        MAX = 0x7FFF_FFFF
-    }
-}
-
-// Index type for `RedDepNodeData`.
-rustc_index::newtype_index! {
-    struct RedDepNodeIndex {
-        MAX = 0x7FFF_FFFF
-    }
-}
-
-// Index type for `LightGreenDepNodeData`.
-rustc_index::newtype_index! {
-    struct LightGreenDepNodeIndex {
-        MAX = 0x7FFF_FFFF
-    }
-}
-
-/// Compressed representation of `HybridIndex` enum. Bits unused by the
-/// contained index types are used to encode which index type it contains.
-#[derive(Copy, Clone)]
-struct CompressedHybridIndex(u32);
-
-impl CompressedHybridIndex {
-    const NEW_TAG: u32 = 0b0000_0000_0000_0000_0000_0000_0000_0000;
-    const RED_TAG: u32 = 0b0100_0000_0000_0000_0000_0000_0000_0000;
-    const LIGHT_GREEN_TAG: u32 = 0b1000_0000_0000_0000_0000_0000_0000_0000;
-    const DARK_GREEN_TAG: u32 = 0b1100_0000_0000_0000_0000_0000_0000_0000;
-
-    const TAG_MASK: u32 = 0b1100_0000_0000_0000_0000_0000_0000_0000;
-    const INDEX_MASK: u32 = !Self::TAG_MASK;
-}
-
-impl From<NewDepNodeIndex> for CompressedHybridIndex {
-    #[inline]
-    fn from(index: NewDepNodeIndex) -> Self {
-        CompressedHybridIndex(Self::NEW_TAG | index.as_u32())
-    }
-}
-
-impl From<RedDepNodeIndex> for CompressedHybridIndex {
-    #[inline]
-    fn from(index: RedDepNodeIndex) -> Self {
-        CompressedHybridIndex(Self::RED_TAG | index.as_u32())
-    }
-}
-
-impl From<LightGreenDepNodeIndex> for CompressedHybridIndex {
-    #[inline]
-    fn from(index: LightGreenDepNodeIndex) -> Self {
-        CompressedHybridIndex(Self::LIGHT_GREEN_TAG | index.as_u32())
-    }
-}
-
-impl From<SerializedDepNodeIndex> for CompressedHybridIndex {
-    #[inline]
-    fn from(index: SerializedDepNodeIndex) -> Self {
-        CompressedHybridIndex(Self::DARK_GREEN_TAG | index.as_u32())
-    }
-}
-
-/// Contains an index into one of several node data collections. Elsewhere, we
-/// store `CompressedHyridIndex` instead of this to save space, but convert to
-/// this type during processing to take advantage of the enum match ergonomics.
-enum HybridIndex {
-    New(NewDepNodeIndex),
-    Red(RedDepNodeIndex),
-    LightGreen(LightGreenDepNodeIndex),
-    DarkGreen(SerializedDepNodeIndex),
-}
-
-impl From<CompressedHybridIndex> for HybridIndex {
-    #[inline]
-    fn from(hybrid_index: CompressedHybridIndex) -> Self {
-        let index = hybrid_index.0 & CompressedHybridIndex::INDEX_MASK;
-
-        match hybrid_index.0 & CompressedHybridIndex::TAG_MASK {
-            CompressedHybridIndex::NEW_TAG => HybridIndex::New(NewDepNodeIndex::from_u32(index)),
-            CompressedHybridIndex::RED_TAG => HybridIndex::Red(RedDepNodeIndex::from_u32(index)),
-            CompressedHybridIndex::LIGHT_GREEN_TAG => {
-                HybridIndex::LightGreen(LightGreenDepNodeIndex::from_u32(index))
-            }
-            CompressedHybridIndex::DARK_GREEN_TAG => {
-                HybridIndex::DarkGreen(SerializedDepNodeIndex::from_u32(index))
-            }
-            _ => unreachable!(),
-        }
-    }
-}
-
 // Index type for `DepNodeData`'s edges.
 rustc_index::newtype_index! {
     struct EdgeIndex { .. }
 }
 
-/// Data for nodes in the current graph, divided into different collections
-/// based on their presence in the previous graph, and if present, their color.
-/// We divide nodes this way because different types of nodes are able to share
-/// more or less data with the previous graph.
-///
-/// To enable more sharing, we distinguish between two kinds of green nodes.
-/// Light green nodes are nodes in the previous graph that have been marked
-/// green because we re-executed their queries and the results were the same as
-/// in the previous session. Dark green nodes are nodes in the previous graph
-/// that have been marked green because we were able to mark all of their
-/// dependencies green.
-///
-/// Both light and dark green nodes can share the dep node and fingerprint with
-/// the previous graph, but for light green nodes, we can't be sure that the
-/// edges may be shared without comparing them against the previous edges, so we
-/// store them directly (an approach in which we compare edges with the previous
-/// edges to see if they can be shared was evaluated, but was not found to be
-/// very profitable).
-///
-/// For dark green nodes, we can share everything with the previous graph, which
-/// is why the `HybridIndex::DarkGreen` enum variant contains the index of the
-/// node in the previous graph, and why we don't have a separate collection for
-/// dark green node data--the collection is the `PreviousDepGraph` itself.
-///
-/// (Note that for dark green nodes, the edges in the previous graph
-/// (`SerializedDepNodeIndex`s) must be converted to edges in the current graph
-/// (`DepNodeIndex`s). `CurrentDepGraph` contains `prev_index_to_index`, which
-/// can perform this conversion. It should always be possible, as by definition,
-/// a dark green node is one whose dependencies from the previous session have
-/// all been marked green--which means `prev_index_to_index` contains them.)
-///
-/// Node data is stored in parallel vectors to eliminate the padding between
-/// elements that would be needed to satisfy alignment requirements of the
-/// structure that would contain all of a node's data. We could group tightly
-/// packing subsets of node data together and use fewer vectors, but for
-/// consistency's sake, we use separate vectors for each piece of data.
-struct DepNodeData<K> {
-    /// Data for nodes not in previous graph.
-    new: NewDepNodeData<K>,
-
-    /// Data for nodes in previous graph that have been marked red.
-    red: RedDepNodeData,
-
-    /// Data for nodes in previous graph that have been marked light green.
-    light_green: LightGreenDepNodeData,
-
-    // Edges for all nodes other than dark-green ones. Edges for each node
-    // occupy a contiguous region of this collection, which a node can reference
-    // using two indices. Storing edges this way rather than using an `EdgesVec`
-    // for each node reduces memory consumption by a not insignificant amount
-    // when compiling large crates. The downside is that we have to copy into
-    // this collection the edges from the `EdgesVec`s that are built up during
-    // query execution. But this is mostly balanced out by the more efficient
-    // implementation of `DepGraph::serialize` enabled by this representation.
-    unshared_edges: IndexVec<EdgeIndex, DepNodeIndex>,
-
-    /// Mapping from `DepNodeIndex` to an index into a collection above.
-    /// Indicates which of the above collections contains a node's data.
-    ///
-    /// This collection is wasteful in time and space during incr-full builds,
-    /// because for those, all nodes are new. However, the waste is relatively
-    /// small, and the maintenance cost of avoiding using this for incr-full
-    /// builds is somewhat high and prone to bugginess. It does not seem worth
-    /// it at the time of this writing, but we may want to revisit the idea.
-    hybrid_indices: IndexVec<DepNodeIndex, CompressedHybridIndex>,
-}
-
-/// Data for nodes not in previous graph. Since we cannot share any data with
-/// the previous graph, so we must store all of such a node's data here.
-struct NewDepNodeData<K> {
-    nodes: IndexVec<NewDepNodeIndex, DepNode<K>>,
-    edges: IndexVec<NewDepNodeIndex, Range<EdgeIndex>>,
-    fingerprints: IndexVec<NewDepNodeIndex, Fingerprint>,
-}
-
-/// Data for nodes in previous graph that have been marked red. We can share the
-/// dep node with the previous graph, but the edges may be different, and the
-/// fingerprint is known to be different, so we store the latter two directly.
-struct RedDepNodeData {
-    node_indices: IndexVec<RedDepNodeIndex, SerializedDepNodeIndex>,
-    edges: IndexVec<RedDepNodeIndex, Range<EdgeIndex>>,
-    fingerprints: IndexVec<RedDepNodeIndex, Fingerprint>,
-}
-
-/// Data for nodes in previous graph that have been marked green because we
-/// re-executed their queries and the results were the same as in the previous
-/// session. We can share the dep node and the fingerprint with the previous
-/// graph, but the edges may be different, so we store them directly.
-struct LightGreenDepNodeData {
-    node_indices: IndexVec<LightGreenDepNodeIndex, SerializedDepNodeIndex>,
-    edges: IndexVec<LightGreenDepNodeIndex, Range<EdgeIndex>>,
-}
-
 /// `CurrentDepGraph` stores the dependency graph for the current session. It
 /// will be populated as we run queries or tasks. We never remove nodes from the
 /// graph: they are only added.
 ///
-/// The nodes in it are identified by a `DepNodeIndex`. Internally, this maps to
-/// a `HybridIndex`, which identifies which collection in the `data` field
-/// contains a node's data. Which collection is used for a node depends on
-/// whether the node was present in the `PreviousDepGraph`, and if so, the color
-/// of the node. Each type of node can share more or less data with the previous
-/// graph. When possible, we can store just the index of the node in the
-/// previous graph, rather than duplicating its data in our own collections.
-/// This is important, because these graph structures are some of the largest in
-/// the compiler.
+/// The nodes in it are identified by a `DepNodeIndex`. We avoid keeping the nodes
+/// in memory.  This is important, because these graph structures are some of the
+/// largest in the compiler.
 ///
-/// For the same reason, we also avoid storing `DepNode`s more than once as map
+/// For this reason, we avoid storing `DepNode`s more than once as map
 /// keys. The `new_node_to_index` map only contains nodes not in the previous
 /// graph, and we map nodes in the previous graph to indices via a two-step
 /// mapping. `PreviousDepGraph` maps from `DepNode` to `SerializedDepNodeIndex`,
@@ -1417,15 +869,15 @@ struct LightGreenDepNodeData {
 /// `new_node_to_index` and `data`, or `prev_index_to_index` and `data`. When
 /// manipulating both, we acquire `new_node_to_index` or `prev_index_to_index`
 /// first, and `data` second.
-pub(super) struct CurrentDepGraph<K> {
-    data: Lock<DepNodeData<K>>,
+pub(super) struct CurrentDepGraph<K: DepKind> {
+    encoder: Steal<GraphEncoder<K>>,
     new_node_to_index: Sharded<FxHashMap<DepNode<K>, DepNodeIndex>>,
     prev_index_to_index: Lock<IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>>,
 
     /// Used to trap when a specific edge is added to the graph.
     /// This is used for debug purposes and is only active with `debug_assertions`.
-    #[allow(dead_code)]
-    forbidden_edge: Option<EdgeFilter>,
+    #[cfg(debug_assertions)]
+    forbidden_edge: Option<EdgeFilter<K>>,
 
     /// Anonymous `DepNode`s are nodes whose IDs we compute from the list of
     /// their edges. This has the beneficial side-effect that multiple anonymous
@@ -1447,7 +899,12 @@ pub(super) struct CurrentDepGraph<K> {
 }
 
 impl<K: DepKind> CurrentDepGraph<K> {
-    fn new(prev_graph_node_count: usize) -> CurrentDepGraph<K> {
+    fn new(
+        prev_graph_node_count: usize,
+        encoder: FileEncoder,
+        record_graph: bool,
+        record_stats: bool,
+    ) -> CurrentDepGraph<K> {
         use std::time::{SystemTime, UNIX_EPOCH};
 
         let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
@@ -1455,70 +912,29 @@ impl<K: DepKind> CurrentDepGraph<K> {
         let mut stable_hasher = StableHasher::new();
         nanos.hash(&mut stable_hasher);
 
-        let forbidden_edge = if cfg!(debug_assertions) {
-            match env::var("RUST_FORBID_DEP_GRAPH_EDGE") {
-                Ok(s) => match EdgeFilter::new(&s) {
-                    Ok(f) => Some(f),
-                    Err(err) => panic!("RUST_FORBID_DEP_GRAPH_EDGE invalid: {}", err),
-                },
-                Err(_) => None,
-            }
-        } else {
-            None
+        #[cfg(debug_assertions)]
+        let forbidden_edge = match env::var("RUST_FORBID_DEP_GRAPH_EDGE") {
+            Ok(s) => match EdgeFilter::new(&s) {
+                Ok(f) => Some(f),
+                Err(err) => panic!("RUST_FORBID_DEP_GRAPH_EDGE invalid: {}", err),
+            },
+            Err(_) => None,
         };
 
-        // Pre-allocate the dep node structures. We over-allocate a little so
-        // that we hopefully don't have to re-allocate during this compilation
-        // session. The over-allocation for new nodes is 2% plus a small
-        // constant to account for the fact that in very small crates 2% might
-        // not be enough. The allocation for red and green node data doesn't
-        // include a constant, as we don't want to allocate anything for these
-        // structures during full incremental builds, where they aren't used.
-        //
-        // These estimates are based on the distribution of node and edge counts
-        // seen in rustc-perf benchmarks, adjusted somewhat to account for the
-        // fact that these benchmarks aren't perfectly representative.
-        //
-        // FIXME Use a collection type that doesn't copy node and edge data and
-        // grow multiplicatively on reallocation. Without such a collection or
-        // solution having the same effect, there is a performance hazard here
-        // in both time and space, as growing these collections means copying a
-        // large amount of data and doubling already large buffer capacities. A
-        // solution for this will also mean that it's less important to get
-        // these estimates right.
-        let new_node_count_estimate = (prev_graph_node_count * 2) / 100 + 200;
-        let red_node_count_estimate = (prev_graph_node_count * 3) / 100;
-        let light_green_node_count_estimate = (prev_graph_node_count * 25) / 100;
-        let total_node_count_estimate = prev_graph_node_count + new_node_count_estimate;
-
-        let average_edges_per_node_estimate = 6;
-        let unshared_edge_count_estimate = average_edges_per_node_estimate
-            * (new_node_count_estimate + red_node_count_estimate + light_green_node_count_estimate);
-
         // We store a large collection of these in `prev_index_to_index` during
         // non-full incremental builds, and want to ensure that the element size
         // doesn't inadvertently increase.
         static_assert_size!(Option<DepNodeIndex>, 4);
 
+        let new_node_count_estimate = 102 * prev_graph_node_count / 100 + 200;
+
         CurrentDepGraph {
-            data: Lock::new(DepNodeData {
-                new: NewDepNodeData {
-                    nodes: IndexVec::with_capacity(new_node_count_estimate),
-                    edges: IndexVec::with_capacity(new_node_count_estimate),
-                    fingerprints: IndexVec::with_capacity(new_node_count_estimate),
-                },
-                red: RedDepNodeData {
-                    node_indices: IndexVec::with_capacity(red_node_count_estimate),
-                    edges: IndexVec::with_capacity(red_node_count_estimate),
-                    fingerprints: IndexVec::with_capacity(red_node_count_estimate),
-                },
-                light_green: LightGreenDepNodeData {
-                    node_indices: IndexVec::with_capacity(light_green_node_count_estimate),
-                    edges: IndexVec::with_capacity(light_green_node_count_estimate),
-                },
-                unshared_edges: IndexVec::with_capacity(unshared_edge_count_estimate),
-                hybrid_indices: IndexVec::with_capacity(total_node_count_estimate),
-            }),
+            encoder: Steal::new(GraphEncoder::new(
+                encoder,
+                prev_graph_node_count,
+                record_graph,
+                record_stats,
+            )),
             new_node_to_index: Sharded::new(|| {
                 FxHashMap::with_capacity_and_hasher(
                     new_node_count_estimate / sharded::SHARDS,
@@ -1527,89 +943,143 @@ impl<K: DepKind> CurrentDepGraph<K> {
             }),
             prev_index_to_index: Lock::new(IndexVec::from_elem_n(None, prev_graph_node_count)),
             anon_id_seed: stable_hasher.finish(),
+            #[cfg(debug_assertions)]
             forbidden_edge,
             total_read_count: AtomicU64::new(0),
             total_duplicate_read_count: AtomicU64::new(0),
         }
     }
 
+    #[cfg(debug_assertions)]
+    fn record_edge(&self, dep_node_index: DepNodeIndex, key: DepNode<K>) {
+        if let Some(forbidden_edge) = &self.forbidden_edge {
+            forbidden_edge.index_to_node.lock().insert(dep_node_index, key);
+        }
+    }
+
+    /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it.
+    /// Assumes that this is a node that has no equivalent in the previous dep-graph.
     fn intern_new_node(
         &self,
-        prev_graph: &PreviousDepGraph<K>,
-        dep_node: DepNode<K>,
+        profiler: &SelfProfilerRef,
+        key: DepNode<K>,
         edges: EdgesVec,
-        fingerprint: Fingerprint,
+        current_fingerprint: Fingerprint,
     ) -> DepNodeIndex {
-        debug_assert!(
-            prev_graph.node_to_index_opt(&dep_node).is_none(),
-            "node in previous graph should be interned using one \
-            of `intern_red_node`, `intern_light_green_node`, etc."
-        );
-
-        match self.new_node_to_index.get_shard_by_value(&dep_node).lock().entry(dep_node) {
+        match self.new_node_to_index.get_shard_by_value(&key).lock().entry(key) {
             Entry::Occupied(entry) => *entry.get(),
             Entry::Vacant(entry) => {
-                let data = &mut *self.data.lock();
-                let new_index = data.new.nodes.push(dep_node);
-                add_edges(&mut data.unshared_edges, &mut data.new.edges, edges);
-                data.new.fingerprints.push(fingerprint);
-                let dep_node_index = data.hybrid_indices.push(new_index.into());
+                let dep_node_index =
+                    self.encoder.borrow().send(profiler, key, current_fingerprint, edges);
                 entry.insert(dep_node_index);
+                #[cfg(debug_assertions)]
+                self.record_edge(dep_node_index, key);
                 dep_node_index
             }
         }
     }
 
-    fn intern_red_node(
+    fn intern_node(
         &self,
+        profiler: &SelfProfilerRef,
         prev_graph: &PreviousDepGraph<K>,
-        prev_index: SerializedDepNodeIndex,
+        key: DepNode<K>,
         edges: EdgesVec,
-        fingerprint: Fingerprint,
-    ) -> DepNodeIndex {
-        self.debug_assert_not_in_new_nodes(prev_graph, prev_index);
+        fingerprint: Option<Fingerprint>,
+        print_status: bool,
+    ) -> (DepNodeIndex, Option<(SerializedDepNodeIndex, DepNodeColor)>) {
+        let print_status = cfg!(debug_assertions) && print_status;
+
+        if let Some(prev_index) = prev_graph.node_to_index_opt(&key) {
+            // Determine the color and index of the new `DepNode`.
+            if let Some(fingerprint) = fingerprint {
+                if fingerprint == prev_graph.fingerprint_by_index(prev_index) {
+                    if print_status {
+                        eprintln!("[task::green] {:?}", key);
+                    }
 
-        let mut prev_index_to_index = self.prev_index_to_index.lock();
+                    // This is a green node: it existed in the previous compilation,
+                    // its query was re-executed, and it has the same result as before.
+                    let mut prev_index_to_index = self.prev_index_to_index.lock();
+
+                    let dep_node_index = match prev_index_to_index[prev_index] {
+                        Some(dep_node_index) => dep_node_index,
+                        None => {
+                            let dep_node_index =
+                                self.encoder.borrow().send(profiler, key, fingerprint, edges);
+                            prev_index_to_index[prev_index] = Some(dep_node_index);
+                            dep_node_index
+                        }
+                    };
 
-        match prev_index_to_index[prev_index] {
-            Some(dep_node_index) => dep_node_index,
-            None => {
-                let data = &mut *self.data.lock();
-                let red_index = data.red.node_indices.push(prev_index);
-                add_edges(&mut data.unshared_edges, &mut data.red.edges, edges);
-                data.red.fingerprints.push(fingerprint);
-                let dep_node_index = data.hybrid_indices.push(red_index.into());
-                prev_index_to_index[prev_index] = Some(dep_node_index);
-                dep_node_index
-            }
-        }
-    }
+                    #[cfg(debug_assertions)]
+                    self.record_edge(dep_node_index, key);
+                    (dep_node_index, Some((prev_index, DepNodeColor::Green(dep_node_index))))
+                } else {
+                    if print_status {
+                        eprintln!("[task::red] {:?}", key);
+                    }
 
-    fn intern_light_green_node(
-        &self,
-        prev_graph: &PreviousDepGraph<K>,
-        prev_index: SerializedDepNodeIndex,
-        edges: EdgesVec,
-    ) -> DepNodeIndex {
-        self.debug_assert_not_in_new_nodes(prev_graph, prev_index);
+                    // This is a red node: it existed in the previous compilation, its query
+                    // was re-executed, but it has a different result from before.
+                    let mut prev_index_to_index = self.prev_index_to_index.lock();
+
+                    let dep_node_index = match prev_index_to_index[prev_index] {
+                        Some(dep_node_index) => dep_node_index,
+                        None => {
+                            let dep_node_index =
+                                self.encoder.borrow().send(profiler, key, fingerprint, edges);
+                            prev_index_to_index[prev_index] = Some(dep_node_index);
+                            dep_node_index
+                        }
+                    };
 
-        let mut prev_index_to_index = self.prev_index_to_index.lock();
+                    #[cfg(debug_assertions)]
+                    self.record_edge(dep_node_index, key);
+                    (dep_node_index, Some((prev_index, DepNodeColor::Red)))
+                }
+            } else {
+                if print_status {
+                    eprintln!("[task::unknown] {:?}", key);
+                }
 
-        match prev_index_to_index[prev_index] {
-            Some(dep_node_index) => dep_node_index,
-            None => {
-                let data = &mut *self.data.lock();
-                let light_green_index = data.light_green.node_indices.push(prev_index);
-                add_edges(&mut data.unshared_edges, &mut data.light_green.edges, edges);
-                let dep_node_index = data.hybrid_indices.push(light_green_index.into());
-                prev_index_to_index[prev_index] = Some(dep_node_index);
-                dep_node_index
+                // This is a red node, effectively: it existed in the previous compilation
+                // session, its query was re-executed, but it doesn't compute a result hash
+                // (i.e. it represents a `no_hash` query), so we have no way of determining
+                // whether or not the result was the same as before.
+                let mut prev_index_to_index = self.prev_index_to_index.lock();
+
+                let dep_node_index = match prev_index_to_index[prev_index] {
+                    Some(dep_node_index) => dep_node_index,
+                    None => {
+                        let dep_node_index =
+                            self.encoder.borrow().send(profiler, key, Fingerprint::ZERO, edges);
+                        prev_index_to_index[prev_index] = Some(dep_node_index);
+                        dep_node_index
+                    }
+                };
+
+                #[cfg(debug_assertions)]
+                self.record_edge(dep_node_index, key);
+                (dep_node_index, Some((prev_index, DepNodeColor::Red)))
+            }
+        } else {
+            if print_status {
+                eprintln!("[task::new] {:?}", key);
             }
+
+            let fingerprint = fingerprint.unwrap_or(Fingerprint::ZERO);
+
+            // This is a new node: it didn't exist in the previous compilation session.
+            let dep_node_index = self.intern_new_node(profiler, key, edges, fingerprint);
+
+            (dep_node_index, None)
         }
     }
 
-    fn intern_dark_green_node(
+    fn promote_node_and_deps_to_current(
         &self,
+        profiler: &SelfProfilerRef,
         prev_graph: &PreviousDepGraph<K>,
         prev_index: SerializedDepNodeIndex,
     ) -> DepNodeIndex {
@@ -1620,9 +1090,20 @@ impl<K: DepKind> CurrentDepGraph<K> {
         match prev_index_to_index[prev_index] {
             Some(dep_node_index) => dep_node_index,
             None => {
-                let mut data = self.data.lock();
-                let dep_node_index = data.hybrid_indices.push(prev_index.into());
+                let key = prev_graph.index_to_node(prev_index);
+                let dep_node_index = self.encoder.borrow().send(
+                    profiler,
+                    key,
+                    prev_graph.fingerprint_by_index(prev_index),
+                    prev_graph
+                        .edge_targets_from(prev_index)
+                        .iter()
+                        .map(|i| prev_index_to_index[*i].unwrap())
+                        .collect(),
+                );
                 prev_index_to_index[prev_index] = Some(dep_node_index);
+                #[cfg(debug_assertions)]
+                self.record_edge(dep_node_index, key);
                 dep_node_index
             }
         }
@@ -1642,18 +1123,6 @@ impl<K: DepKind> CurrentDepGraph<K> {
     }
 }
 
-#[inline]
-fn add_edges<I: Idx>(
-    edges: &mut IndexVec<EdgeIndex, DepNodeIndex>,
-    edge_indices: &mut IndexVec<I, Range<EdgeIndex>>,
-    new_edges: EdgesVec,
-) {
-    let start = edges.next_index();
-    edges.extend(new_edges);
-    let end = edges.next_index();
-    edge_indices.push(start..end);
-}
-
 /// The capacity of the `reads` field `SmallVec`
 const TASK_DEPS_READS_CAP: usize = 8;
 type EdgesVec = SmallVec<[DepNodeIndex; TASK_DEPS_READS_CAP]>;
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index e8fb71be3e0..1b6ecf3e637 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -13,6 +13,7 @@ pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};
 
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sync::Lock;
+use rustc_serialize::{opaque::FileEncoder, Encodable};
 use rustc_session::Session;
 
 use std::fmt;
@@ -59,7 +60,7 @@ impl<T: DepContext> HasDepContext for T {
 }
 
 /// Describe the different families of dependency nodes.
-pub trait DepKind: Copy + fmt::Debug + Eq + Hash {
+pub trait DepKind: Copy + fmt::Debug + Eq + Hash + Send + Encodable<FileEncoder> + 'static {
     const NULL: Self;
 
     /// Return whether this kind always require evaluation.
diff --git a/compiler/rustc_query_system/src/dep_graph/prev.rs b/compiler/rustc_query_system/src/dep_graph/prev.rs
index c3d0f795255..6303bbf53b9 100644
--- a/compiler/rustc_query_system/src/dep_graph/prev.rs
+++ b/compiler/rustc_query_system/src/dep_graph/prev.rs
@@ -36,11 +36,6 @@ impl<K: DepKind> PreviousDepGraph<K> {
     }
 
     #[inline]
-    pub fn node_to_index(&self, dep_node: &DepNode<K>) -> SerializedDepNodeIndex {
-        self.index[dep_node]
-    }
-
-    #[inline]
     pub fn node_to_index_opt(&self, dep_node: &DepNode<K>) -> Option<SerializedDepNodeIndex> {
         self.index.get(dep_node).cloned()
     }
diff --git a/compiler/rustc_query_system/src/dep_graph/query.rs b/compiler/rustc_query_system/src/dep_graph/query.rs
index cc25d08cb54..27b3b5e1366 100644
--- a/compiler/rustc_query_system/src/dep_graph/query.rs
+++ b/compiler/rustc_query_system/src/dep_graph/query.rs
@@ -1,38 +1,43 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph::implementation::{Direction, Graph, NodeIndex, INCOMING};
+use rustc_index::vec::IndexVec;
 
-use super::{DepKind, DepNode};
+use super::{DepKind, DepNode, DepNodeIndex};
 
 pub struct DepGraphQuery<K> {
     pub graph: Graph<DepNode<K>, ()>,
     pub indices: FxHashMap<DepNode<K>, NodeIndex>,
+    pub dep_index_to_index: IndexVec<DepNodeIndex, Option<NodeIndex>>,
 }
 
 impl<K: DepKind> DepGraphQuery<K> {
-    pub fn new(
-        nodes: &[DepNode<K>],
-        edge_list_indices: &[(usize, usize)],
-        edge_list_data: &[usize],
-    ) -> DepGraphQuery<K> {
-        let mut graph = Graph::with_capacity(nodes.len(), edge_list_data.len());
-        let mut indices = FxHashMap::default();
-        for node in nodes {
-            indices.insert(*node, graph.add_node(*node));
-        }
+    pub fn new(prev_node_count: usize) -> DepGraphQuery<K> {
+        let node_count = prev_node_count + prev_node_count / 4;
+        let edge_count = 6 * node_count;
 
-        for (source, &(start, end)) in edge_list_indices.iter().enumerate() {
-            for &target in &edge_list_data[start..end] {
-                let source = indices[&nodes[source]];
-                let target = indices[&nodes[target]];
-                graph.add_edge(source, target, ());
-            }
-        }
+        let graph = Graph::with_capacity(node_count, edge_count);
+        let indices = FxHashMap::default();
+        let dep_index_to_index = IndexVec::new();
 
-        DepGraphQuery { graph, indices }
+        DepGraphQuery { graph, indices, dep_index_to_index }
     }
 
-    pub fn contains_node(&self, node: &DepNode<K>) -> bool {
-        self.indices.contains_key(&node)
+    pub fn push(&mut self, index: DepNodeIndex, node: DepNode<K>, edges: &[DepNodeIndex]) {
+        let source = self.graph.add_node(node);
+        if index.index() >= self.dep_index_to_index.len() {
+            self.dep_index_to_index.resize(index.index() + 1, None);
+        }
+        self.dep_index_to_index[index] = Some(source);
+        self.indices.insert(node, source);
+
+        for &target in edges.iter() {
+            let target = self.dep_index_to_index[target];
+            // We may miss the edges that are pushed while the `DepGraphQuery` is being accessed.
+            // Skip them to issues.
+            if let Some(target) = target {
+                self.graph.add_edge(source, target, ());
+            }
+        }
     }
 
     pub fn nodes(&self) -> Vec<&DepNode<K>> {
diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs
index 9bb922b0a90..6f3d1fb7199 100644
--- a/compiler/rustc_query_system/src/dep_graph/serialized.rs
+++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs
@@ -1,9 +1,28 @@
 //! The data that we will serialize and deserialize.
+//!
+//! The dep-graph is serialized as a sequence of NodeInfo, with the dependencies
+//! specified inline.  The total number of nodes and edges are stored as the last
+//! 16 bytes of the file, so we can find them easily at decoding time.
+//!
+//! The serialisation is performed on-demand when each node is emitted. Using this
+//! scheme, we do not need to keep the current graph in memory.
+//!
+//! The deserisalisation is performed manually, in order to convert from the stored
+//! sequence of NodeInfos to the different arrays in SerializedDepGraph.  Since the
+//! node and edge count are stored at the end of the file, all the arrays can be
+//! pre-allocated with the right length.
 
-use super::{DepKind, DepNode};
+use super::query::DepGraphQuery;
+use super::{DepKind, DepNode, DepNodeIndex};
 use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_index::vec::IndexVec;
-use rustc_serialize::{Decodable, Decoder};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::profiling::SelfProfilerRef;
+use rustc_data_structures::sync::Lock;
+use rustc_index::vec::{Idx, IndexVec};
+use rustc_serialize::opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize};
+use rustc_serialize::{Decodable, Decoder, Encodable};
+use smallvec::SmallVec;
+use std::convert::TryInto;
 
 // The maximum value of `SerializedDepNodeIndex` leaves the upper two bits
 // unused so that we can store multiple index types in `CompressedHybridIndex`,
@@ -50,78 +69,239 @@ impl<K: DepKind> SerializedDepGraph<K> {
     }
 }
 
-impl<D: Decoder, K: DepKind + Decodable<D>> Decodable<D> for SerializedDepGraph<K> {
-    fn decode(d: &mut D) -> Result<SerializedDepGraph<K>, D::Error> {
-        // We used to serialize the dep graph by creating and serializing a `SerializedDepGraph`
-        // using data copied from the `DepGraph`. But copying created a large memory spike, so we
-        // now serialize directly from the `DepGraph` as if it's a `SerializedDepGraph`. Because we
-        // deserialize that data into a `SerializedDepGraph` in the next compilation session, we
-        // need `DepGraph`'s `Encodable` and `SerializedDepGraph`'s `Decodable` implementations to
-        // be in sync. If you update this decoding, be sure to update the encoding, and vice-versa.
-        //
-        // We mimic the sequence of `Encode` and `Encodable` method calls used by the `DepGraph`'s
-        // `Encodable` implementation with the corresponding sequence of `Decode` and `Decodable`
-        // method calls. E.g. `Decode::read_struct` pairs with `Encode::emit_struct`, `DepNode`'s
-        // `decode` pairs with `DepNode`'s `encode`, and so on. Any decoding methods not associated
-        // with corresponding encoding methods called in `DepGraph`'s `Encodable` implementation
-        // are off limits, because we'd be relying on their implementation details.
-        //
-        // For example, because we know it happens to do the right thing, its tempting to just use
-        // `IndexVec`'s `Decodable` implementation to decode into some of the collections below,
-        // even though `DepGraph` doesn't use its `Encodable` implementation. But the `IndexVec`
-        // implementation could change, and we'd have a bug.
-        //
-        // Variables below are explicitly typed so that anyone who changes the `SerializedDepGraph`
-        // representation without updating this function will encounter a compilation error, and
-        // know to update this and possibly the `DepGraph` `Encodable` implementation accordingly
-        // (the latter should serialize data in a format compatible with our representation).
-
-        d.read_struct("SerializedDepGraph", 4, |d| {
-            let nodes: IndexVec<SerializedDepNodeIndex, DepNode<K>> =
-                d.read_struct_field("nodes", 0, |d| {
-                    d.read_seq(|d, len| {
-                        let mut v = IndexVec::with_capacity(len);
-                        for i in 0..len {
-                            v.push(d.read_seq_elt(i, |d| Decodable::decode(d))?);
-                        }
-                        Ok(v)
-                    })
-                })?;
+impl<'a, K: DepKind + Decodable<opaque::Decoder<'a>>> Decodable<opaque::Decoder<'a>>
+    for SerializedDepGraph<K>
+{
+    #[instrument(skip(d))]
+    fn decode(d: &mut opaque::Decoder<'a>) -> Result<SerializedDepGraph<K>, String> {
+        let start_position = d.position();
 
-            let fingerprints: IndexVec<SerializedDepNodeIndex, Fingerprint> =
-                d.read_struct_field("fingerprints", 1, |d| {
-                    d.read_seq(|d, len| {
-                        let mut v = IndexVec::with_capacity(len);
-                        for i in 0..len {
-                            v.push(d.read_seq_elt(i, |d| Decodable::decode(d))?);
-                        }
-                        Ok(v)
-                    })
-                })?;
+        // The last 16 bytes are the node count and edge count.
+        debug!("position: {:?}", d.position());
+        d.set_position(d.data.len() - 2 * IntEncodedWithFixedSize::ENCODED_SIZE);
+        debug!("position: {:?}", d.position());
 
-            let edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)> = d
-                .read_struct_field("edge_list_indices", 2, |d| {
-                    d.read_seq(|d, len| {
-                        let mut v = IndexVec::with_capacity(len);
-                        for i in 0..len {
-                            v.push(d.read_seq_elt(i, |d| Decodable::decode(d))?);
-                        }
-                        Ok(v)
-                    })
-                })?;
+        let node_count = IntEncodedWithFixedSize::decode(d)?.0 as usize;
+        let edge_count = IntEncodedWithFixedSize::decode(d)?.0 as usize;
+        debug!(?node_count, ?edge_count);
+
+        debug!("position: {:?}", d.position());
+        d.set_position(start_position);
+        debug!("position: {:?}", d.position());
+
+        let mut nodes = IndexVec::with_capacity(node_count);
+        let mut fingerprints = IndexVec::with_capacity(node_count);
+        let mut edge_list_indices = IndexVec::with_capacity(node_count);
+        let mut edge_list_data = Vec::with_capacity(edge_count);
+
+        for _index in 0..node_count {
+            d.read_struct("NodeInfo", 3, |d| {
+                let dep_node: DepNode<K> = d.read_struct_field("node", 0, Decodable::decode)?;
+                let _i: SerializedDepNodeIndex = nodes.push(dep_node);
+                debug_assert_eq!(_i.index(), _index);
 
-            let edge_list_data: Vec<SerializedDepNodeIndex> =
-                d.read_struct_field("edge_list_data", 3, |d| {
+                let fingerprint: Fingerprint =
+                    d.read_struct_field("fingerprint", 1, Decodable::decode)?;
+                let _i: SerializedDepNodeIndex = fingerprints.push(fingerprint);
+                debug_assert_eq!(_i.index(), _index);
+
+                d.read_struct_field("edges", 2, |d| {
                     d.read_seq(|d, len| {
-                        let mut v = Vec::with_capacity(len);
-                        for i in 0..len {
-                            v.push(d.read_seq_elt(i, |d| Decodable::decode(d))?);
+                        let start = edge_list_data.len().try_into().unwrap();
+                        for e in 0..len {
+                            let edge = d.read_seq_elt(e, Decodable::decode)?;
+                            edge_list_data.push(edge);
                         }
-                        Ok(v)
+                        let end = edge_list_data.len().try_into().unwrap();
+                        let _i: SerializedDepNodeIndex = edge_list_indices.push((start, end));
+                        debug_assert_eq!(_i.index(), _index);
+                        Ok(())
                     })
-                })?;
+                })
+            })?;
+        }
+
+        Ok(SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data })
+    }
+}
+
+#[derive(Debug, Encodable, Decodable)]
+pub struct NodeInfo<K: DepKind> {
+    node: DepNode<K>,
+    fingerprint: Fingerprint,
+    edges: SmallVec<[DepNodeIndex; 8]>,
+}
+
+struct Stat<K: DepKind> {
+    kind: K,
+    node_counter: u64,
+    edge_counter: u64,
+}
+
+struct EncoderState<K: DepKind> {
+    encoder: FileEncoder,
+    total_node_count: usize,
+    total_edge_count: usize,
+    result: FileEncodeResult,
+    stats: Option<FxHashMap<K, Stat<K>>>,
+}
+
+impl<K: DepKind> EncoderState<K> {
+    fn new(encoder: FileEncoder, record_stats: bool) -> Self {
+        Self {
+            encoder,
+            total_edge_count: 0,
+            total_node_count: 0,
+            result: Ok(()),
+            stats: if record_stats { Some(FxHashMap::default()) } else { None },
+        }
+    }
+
+    #[instrument(skip(self, record_graph))]
+    fn encode_node(
+        &mut self,
+        node: &NodeInfo<K>,
+        record_graph: &Option<Lock<DepGraphQuery<K>>>,
+    ) -> DepNodeIndex {
+        let index = DepNodeIndex::new(self.total_node_count);
+        self.total_node_count += 1;
+
+        let edge_count = node.edges.len();
+        self.total_edge_count += edge_count;
+
+        if let Some(record_graph) = &record_graph {
+            // Do not ICE when a query is called from within `with_query`.
+            if let Some(record_graph) = &mut record_graph.try_lock() {
+                record_graph.push(index, node.node, &node.edges);
+            }
+        }
+
+        if let Some(stats) = &mut self.stats {
+            let kind = node.node.kind;
+
+            let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
+            stat.node_counter += 1;
+            stat.edge_counter += edge_count as u64;
+        }
+
+        debug!(?index, ?node);
+        let encoder = &mut self.encoder;
+        if self.result.is_ok() {
+            self.result = node.encode(encoder);
+        }
+        index
+    }
+
+    fn finish(self) -> FileEncodeResult {
+        let Self { mut encoder, total_node_count, total_edge_count, result, stats: _ } = self;
+        let () = result?;
+
+        let node_count = total_node_count.try_into().unwrap();
+        let edge_count = total_edge_count.try_into().unwrap();
+
+        debug!(?node_count, ?edge_count);
+        debug!("position: {:?}", encoder.position());
+        IntEncodedWithFixedSize(node_count).encode(&mut encoder)?;
+        IntEncodedWithFixedSize(edge_count).encode(&mut encoder)?;
+        debug!("position: {:?}", encoder.position());
+        // Drop the encoder so that nothing is written after the counts.
+        encoder.flush()
+    }
+}
+
+pub struct GraphEncoder<K: DepKind> {
+    status: Lock<EncoderState<K>>,
+    record_graph: Option<Lock<DepGraphQuery<K>>>,
+}
+
+impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
+    pub fn new(
+        encoder: FileEncoder,
+        prev_node_count: usize,
+        record_graph: bool,
+        record_stats: bool,
+    ) -> Self {
+        let record_graph =
+            if record_graph { Some(Lock::new(DepGraphQuery::new(prev_node_count))) } else { None };
+        let status = Lock::new(EncoderState::new(encoder, record_stats));
+        GraphEncoder { status, record_graph }
+    }
+
+    pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery<K>)) {
+        if let Some(record_graph) = &self.record_graph {
+            f(&record_graph.lock())
+        }
+    }
+
+    pub(crate) fn print_incremental_info(
+        &self,
+        total_read_count: u64,
+        total_duplicate_read_count: u64,
+    ) {
+        let status = self.status.lock();
+        if let Some(record_stats) = &status.stats {
+            let mut stats: Vec<_> = record_stats.values().collect();
+            stats.sort_by_key(|s| -(s.node_counter as i64));
+
+            const SEPARATOR: &str = "[incremental] --------------------------------\
+                                     ----------------------------------------------\
+                                     ------------";
+
+            eprintln!("[incremental]");
+            eprintln!("[incremental] DepGraph Statistics");
+            eprintln!("{}", SEPARATOR);
+            eprintln!("[incremental]");
+            eprintln!("[incremental] Total Node Count: {}", status.total_node_count);
+            eprintln!("[incremental] Total Edge Count: {}", status.total_edge_count);
+
+            if cfg!(debug_assertions) {
+                eprintln!("[incremental] Total Edge Reads: {}", total_read_count);
+                eprintln!(
+                    "[incremental] Total Duplicate Edge Reads: {}",
+                    total_duplicate_read_count
+                );
+            }
+
+            eprintln!("[incremental]");
+            eprintln!(
+                "[incremental]  {:<36}| {:<17}| {:<12}| {:<17}|",
+                "Node Kind", "Node Frequency", "Node Count", "Avg. Edge Count"
+            );
+            eprintln!("{}", SEPARATOR);
+
+            for stat in stats {
+                let node_kind_ratio =
+                    (100.0 * (stat.node_counter as f64)) / (status.total_node_count as f64);
+                let node_kind_avg_edges = (stat.edge_counter as f64) / (stat.node_counter as f64);
+
+                eprintln!(
+                    "[incremental]  {:<36}|{:>16.1}% |{:>12} |{:>17.1} |",
+                    format!("{:?}", stat.kind),
+                    node_kind_ratio,
+                    stat.node_counter,
+                    node_kind_avg_edges,
+                );
+            }
+
+            eprintln!("{}", SEPARATOR);
+            eprintln!("[incremental]");
+        }
+    }
+
+    pub(crate) fn send(
+        &self,
+        profiler: &SelfProfilerRef,
+        node: DepNode<K>,
+        fingerprint: Fingerprint,
+        edges: SmallVec<[DepNodeIndex; 8]>,
+    ) -> DepNodeIndex {
+        let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph");
+        let node = NodeInfo { node, fingerprint, edges };
+        self.status.lock().encode_node(&node, &self.record_graph)
+    }
 
-            Ok(SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data })
-        })
+    pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
+        let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph");
+        self.status.into_inner().finish()
     }
 }
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index 26b76a9c006..071144f38e7 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -2,7 +2,9 @@
 #![feature(const_fn)]
 #![feature(const_panic)]
 #![feature(core_intrinsics)]
+#![feature(drain_filter)]
 #![feature(hash_raw_entry)]
+#![feature(iter_zip)]
 #![feature(min_specialization)]
 #![feature(stmt_expr_attributes)]
 
diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index ec71c868580..001bf3b216b 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -49,12 +49,11 @@ pub trait QueryCache: QueryStorage {
         index: DepNodeIndex,
     ) -> Self::Stored;
 
-    fn iter<R, L>(
+    fn iter<R>(
         &self,
-        shards: &Sharded<L>,
-        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+        shards: &Sharded<Self::Sharded>,
         f: impl for<'a> FnOnce(
-            Box<dyn Iterator<Item = (&'a Self::Key, &'a Self::Value, DepNodeIndex)> + 'a>,
+            &'a mut dyn Iterator<Item = (&'a Self::Key, &'a Self::Value, DepNodeIndex)>,
         ) -> R,
     ) -> R;
 }
@@ -125,16 +124,14 @@ where
         value
     }
 
-    fn iter<R, L>(
+    fn iter<R>(
         &self,
-        shards: &Sharded<L>,
-        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
-        f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+        shards: &Sharded<Self::Sharded>,
+        f: impl for<'a> FnOnce(&'a mut dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)>) -> R,
     ) -> R {
-        let mut shards = shards.lock_shards();
-        let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
-        let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
-        f(Box::new(results))
+        let shards = shards.lock_shards();
+        let mut results = shards.iter().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
+        f(&mut results)
     }
 }
 
@@ -210,15 +207,13 @@ where
         &value.0
     }
 
-    fn iter<R, L>(
+    fn iter<R>(
         &self,
-        shards: &Sharded<L>,
-        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
-        f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+        shards: &Sharded<Self::Sharded>,
+        f: impl for<'a> FnOnce(&'a mut dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)>) -> R,
     ) -> R {
-        let mut shards = shards.lock_shards();
-        let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
-        let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
-        f(Box::new(results))
+        let shards = shards.lock_shards();
+        let mut results = shards.iter().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
+        f(&mut results)
     }
 }
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index 35a2ac865f2..21f580db04f 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -22,7 +22,7 @@ use {
     rustc_data_structures::{jobserver, OnDrop},
     rustc_rayon_core as rayon_core,
     rustc_span::DUMMY_SP,
-    std::iter::FromIterator,
+    std::iter::{self, FromIterator},
     std::{mem, process},
 };
 
@@ -463,7 +463,7 @@ fn remove_cycle<D: DepKind>(
         spans.rotate_right(1);
 
         // Zip them back together
-        let mut stack: Vec<_> = spans.into_iter().zip(queries).collect();
+        let mut stack: Vec<_> = iter::zip(spans, queries).collect();
 
         // Remove the queries in our cycle from the list of jobs to look at
         for r in &stack {
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 0050670b338..fb8a53048fa 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -76,10 +76,10 @@ impl<C: QueryCache> QueryCacheStore<C> {
     pub fn iter_results<R>(
         &self,
         f: impl for<'a> FnOnce(
-            Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
+            &'a mut dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)>,
         ) -> R,
     ) -> R {
-        self.cache.iter(&self.shards, |shard| &mut *shard, f)
+        self.cache.iter(&self.shards, f)
     }
 }
 
@@ -449,9 +449,11 @@ where
 
         let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
             tcx.start_query(job.id, diagnostics, || {
-                tcx.dep_context()
-                    .dep_graph()
-                    .with_anon_task(query.dep_kind, || query.compute(tcx, key))
+                tcx.dep_context().dep_graph().with_anon_task(
+                    *tcx.dep_context(),
+                    query.dep_kind,
+                    || query.compute(tcx, key),
+                )
             })
         });
 
@@ -533,7 +535,13 @@ where
         None
     };
 
-    let result = if let Some(result) = result {
+    if let Some(result) = result {
+        // If `-Zincremental-verify-ich` is specified, re-hash results from
+        // the cache and make sure that they have the expected fingerprint.
+        if unlikely!(tcx.dep_context().sess().opts.debugging_opts.incremental_verify_ich) {
+            incremental_verify_ich(*tcx.dep_context(), &result, dep_node, query);
+        }
+
         result
     } else {
         // We could not load a result from the on-disk cache, so
@@ -545,32 +553,31 @@ where
 
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
-        result
-    };
+        // Verify that re-running the query produced a result with the expected hash
+        // This catches bugs in query implementations, turning them into ICEs.
+        // For example, a query might sort its result by `DefId` - since `DefId`s are
+        // not stable across compilation sessions, the result could get up getting sorted
+        // in a different order when the query is re-run, even though all of the inputs
+        // (e.g. `DefPathHash` values) were green.
+        //
+        // See issue #82920 for an example of a miscompilation that would get turned into
+        // an ICE by this check
+        incremental_verify_ich(*tcx.dep_context(), &result, dep_node, query);
 
-    // If `-Zincremental-verify-ich` is specified, re-hash results from
-    // the cache and make sure that they have the expected fingerprint.
-    if unlikely!(tcx.dep_context().sess().opts.debugging_opts.incremental_verify_ich) {
-        incremental_verify_ich(*tcx.dep_context(), &result, dep_node, dep_node_index, query);
+        result
     }
-
-    result
 }
 
-#[inline(never)]
-#[cold]
 fn incremental_verify_ich<CTX, K, V: Debug>(
     tcx: CTX::DepContext,
     result: &V,
     dep_node: &DepNode<CTX::DepKind>,
-    dep_node_index: DepNodeIndex,
     query: &QueryVtable<CTX, K, V>,
 ) where
     CTX: QueryContext,
 {
     assert!(
-        Some(tcx.dep_graph().fingerprint_of(dep_node_index))
-            == tcx.dep_graph().prev_fingerprint_of(dep_node),
+        tcx.dep_graph().is_green(dep_node),
         "fingerprint for green query instance not loaded from cache: {:?}",
         dep_node,
     );
@@ -581,9 +588,15 @@ fn incremental_verify_ich<CTX, K, V: Debug>(
     let new_hash = query.hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
     debug!("END verify_ich({:?})", dep_node);
 
-    let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
+    let old_hash = tcx.dep_graph().prev_fingerprint_of(dep_node);
 
-    assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,);
+    assert_eq!(
+        Some(new_hash),
+        old_hash,
+        "found unstable fingerprints for {:?}: {:?}",
+        dep_node,
+        result
+    );
 }
 
 fn force_query_with_job<C, CTX>(
diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index 65e5b0dddea..d77022a65e4 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -1426,19 +1426,19 @@ impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> {
         }
     }
 
-    fn visit_field(&mut self, f: &'b ast::Field) {
+    fn visit_expr_field(&mut self, f: &'b ast::ExprField) {
         if f.is_placeholder {
             self.visit_invoc(f.id);
         } else {
-            visit::walk_field(self, f);
+            visit::walk_expr_field(self, f);
         }
     }
 
-    fn visit_field_pattern(&mut self, fp: &'b ast::FieldPat) {
+    fn visit_pat_field(&mut self, fp: &'b ast::PatField) {
         if fp.is_placeholder {
             self.visit_invoc(fp.id);
         } else {
-            visit::walk_field_pattern(self, fp);
+            visit::walk_pat_field(self, fp);
         }
     }
 
@@ -1458,13 +1458,13 @@ impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> {
         }
     }
 
-    fn visit_struct_field(&mut self, sf: &'b ast::StructField) {
+    fn visit_field_def(&mut self, sf: &'b ast::FieldDef) {
         if sf.is_placeholder {
             self.visit_invoc(sf.id);
         } else {
             let vis = self.resolve_visibility(&sf.vis);
             self.r.visibilities.insert(self.r.local_def_id(sf.id), vis);
-            visit::walk_struct_field(self, sf);
+            visit::walk_field_def(self, sf);
         }
     }
 
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 727d6ab53d8..17f0c39e397 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -1,4 +1,4 @@
-use crate::Resolver;
+use crate::{ImplTraitContext, Resolver};
 use rustc_ast::visit::{self, FnKind};
 use rustc_ast::walk_list;
 use rustc_ast::*;
@@ -16,14 +16,15 @@ crate fn collect_definitions(
     fragment: &AstFragment,
     expansion: ExpnId,
 ) {
-    let parent_def = resolver.invocation_parents[&expansion];
-    fragment.visit_with(&mut DefCollector { resolver, parent_def, expansion });
+    let (parent_def, impl_trait_context) = resolver.invocation_parents[&expansion];
+    fragment.visit_with(&mut DefCollector { resolver, parent_def, expansion, impl_trait_context });
 }
 
 /// Creates `DefId`s for nodes in the AST.
 struct DefCollector<'a, 'b> {
     resolver: &'a mut Resolver<'b>,
     parent_def: LocalDefId,
+    impl_trait_context: ImplTraitContext,
     expansion: ExpnId,
 }
 
@@ -40,7 +41,17 @@ impl<'a, 'b> DefCollector<'a, 'b> {
         self.parent_def = orig_parent_def;
     }
 
-    fn collect_field(&mut self, field: &'a StructField, index: Option<usize>) {
+    fn with_impl_trait<F: FnOnce(&mut Self)>(
+        &mut self,
+        impl_trait_context: ImplTraitContext,
+        f: F,
+    ) {
+        let orig_itc = std::mem::replace(&mut self.impl_trait_context, impl_trait_context);
+        f(self);
+        self.impl_trait_context = orig_itc;
+    }
+
+    fn collect_field(&mut self, field: &'a FieldDef, index: Option<usize>) {
         let index = |this: &Self| {
             index.unwrap_or_else(|| {
                 let node_id = NodeId::placeholder_from_expn_id(this.expansion);
@@ -55,13 +66,14 @@ impl<'a, 'b> DefCollector<'a, 'b> {
         } else {
             let name = field.ident.map_or_else(|| sym::integer(index(self)), |ident| ident.name);
             let def = self.create_def(field.id, DefPathData::ValueNs(name), field.span);
-            self.with_parent(def, |this| visit::walk_struct_field(this, field));
+            self.with_parent(def, |this| visit::walk_field_def(this, field));
         }
     }
 
     fn visit_macro_invoc(&mut self, id: NodeId) {
+        let id = id.placeholder_to_expn_id();
         let old_parent =
-            self.resolver.invocation_parents.insert(id.placeholder_to_expn_id(), self.parent_def);
+            self.resolver.invocation_parents.insert(id, (self.parent_def, self.impl_trait_context));
         assert!(old_parent.is_none(), "parent `LocalDefId` is reset for an invocation");
     }
 }
@@ -103,29 +115,37 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> {
         let def = self.create_def(i.id, def_data, i.span);
 
         self.with_parent(def, |this| {
-            match i.kind {
-                ItemKind::Struct(ref struct_def, _) | ItemKind::Union(ref struct_def, _) => {
-                    // If this is a unit or tuple-like struct, register the constructor.
-                    if let Some(ctor_hir_id) = struct_def.ctor_id() {
-                        this.create_def(ctor_hir_id, DefPathData::Ctor, i.span);
+            this.with_impl_trait(ImplTraitContext::Existential, |this| {
+                match i.kind {
+                    ItemKind::Struct(ref struct_def, _) | ItemKind::Union(ref struct_def, _) => {
+                        // If this is a unit or tuple-like struct, register the constructor.
+                        if let Some(ctor_hir_id) = struct_def.ctor_id() {
+                            this.create_def(ctor_hir_id, DefPathData::Ctor, i.span);
+                        }
                     }
+                    _ => {}
                 }
-                _ => {}
-            }
-            visit::walk_item(this, i);
+                visit::walk_item(this, i);
+            })
         });
     }
 
     fn visit_fn(&mut self, fn_kind: FnKind<'a>, span: Span, _: NodeId) {
         if let FnKind::Fn(_, _, sig, _, body) = fn_kind {
             if let Async::Yes { closure_id, return_impl_trait_id, .. } = sig.header.asyncness {
-                self.create_def(return_impl_trait_id, DefPathData::ImplTrait, span);
+                let return_impl_trait_id =
+                    self.create_def(return_impl_trait_id, DefPathData::ImplTrait, span);
 
                 // For async functions, we need to create their inner defs inside of a
                 // closure to match their desugared representation. Besides that,
                 // we must mirror everything that `visit::walk_fn` below does.
                 self.visit_fn_header(&sig.header);
-                visit::walk_fn_decl(self, &sig.decl);
+                for param in &sig.decl.inputs {
+                    self.visit_param(param);
+                }
+                self.with_parent(return_impl_trait_id, |this| {
+                    this.visit_fn_ret_ty(&sig.decl.output)
+                });
                 let closure_def = self.create_def(closure_id, DefPathData::ClosureExpr, span);
                 self.with_parent(closure_def, |this| walk_list!(this, visit_block, body));
                 return;
@@ -137,6 +157,14 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> {
 
     fn visit_use_tree(&mut self, use_tree: &'a UseTree, id: NodeId, _nested: bool) {
         self.create_def(id, DefPathData::Misc, use_tree.span);
+        match use_tree.kind {
+            UseTreeKind::Simple(_, id1, id2) => {
+                self.create_def(id1, DefPathData::Misc, use_tree.prefix.span);
+                self.create_def(id2, DefPathData::Misc, use_tree.prefix.span);
+            }
+            UseTreeKind::Glob => (),
+            UseTreeKind::Nested(..) => {}
+        }
         visit::walk_use_tree(self, use_tree, id);
     }
 
@@ -191,7 +219,15 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> {
         };
         self.create_def(param.id, def_path_data, param.ident.span);
 
-        visit::walk_generic_param(self, param);
+        // impl-Trait can happen inside generic parameters, like
+        // ```
+        // fn foo<U: Iterator<Item = impl Clone>>() {}
+        // ```
+        //
+        // In that case, the impl-trait is lowered as an additional generic parameter.
+        self.with_impl_trait(ImplTraitContext::Universal(self.parent_def), |this| {
+            visit::walk_generic_param(this, param)
+        });
     }
 
     fn visit_assoc_item(&mut self, i: &'a AssocItem, ctxt: visit::AssocCtxt) {
@@ -244,8 +280,19 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> {
         match ty.kind {
             TyKind::MacCall(..) => self.visit_macro_invoc(ty.id),
             TyKind::ImplTrait(node_id, _) => {
-                let parent_def = self.create_def(node_id, DefPathData::ImplTrait, ty.span);
-                self.with_parent(parent_def, |this| visit::walk_ty(this, ty));
+                let parent_def = match self.impl_trait_context {
+                    ImplTraitContext::Universal(item_def) => self.resolver.create_def(
+                        item_def,
+                        node_id,
+                        DefPathData::ImplTrait,
+                        self.expansion,
+                        ty.span,
+                    ),
+                    ImplTraitContext::Existential => {
+                        self.create_def(node_id, DefPathData::ImplTrait, ty.span)
+                    }
+                };
+                self.with_parent(parent_def, |this| visit::walk_ty(this, ty))
             }
             _ => visit::walk_ty(self, ty),
         }
@@ -262,25 +309,35 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> {
         if arm.is_placeholder { self.visit_macro_invoc(arm.id) } else { visit::walk_arm(self, arm) }
     }
 
-    fn visit_field(&mut self, f: &'a Field) {
-        if f.is_placeholder { self.visit_macro_invoc(f.id) } else { visit::walk_field(self, f) }
+    fn visit_expr_field(&mut self, f: &'a ExprField) {
+        if f.is_placeholder {
+            self.visit_macro_invoc(f.id)
+        } else {
+            visit::walk_expr_field(self, f)
+        }
     }
 
-    fn visit_field_pattern(&mut self, fp: &'a FieldPat) {
+    fn visit_pat_field(&mut self, fp: &'a PatField) {
         if fp.is_placeholder {
             self.visit_macro_invoc(fp.id)
         } else {
-            visit::walk_field_pattern(self, fp)
+            visit::walk_pat_field(self, fp)
         }
     }
 
     fn visit_param(&mut self, p: &'a Param) {
-        if p.is_placeholder { self.visit_macro_invoc(p.id) } else { visit::walk_param(self, p) }
+        if p.is_placeholder {
+            self.visit_macro_invoc(p.id)
+        } else {
+            self.with_impl_trait(ImplTraitContext::Universal(self.parent_def), |this| {
+                visit::walk_param(this, p)
+            })
+        }
     }
 
     // This method is called only when we are visiting an individual field
     // after expanding an attribute on it.
-    fn visit_struct_field(&mut self, field: &'a StructField) {
+    fn visit_field_def(&mut self, field: &'a FieldDef) {
         self.collect_field(field, None);
     }
 }
diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
index 7493fd68505..87e28f7fcc5 100644
--- a/compiler/rustc_resolve/src/diagnostics.rs
+++ b/compiler/rustc_resolve/src/diagnostics.rs
@@ -450,12 +450,12 @@ impl<'a> Resolver<'a> {
                     self.session,
                     span,
                     E0128,
-                    "type parameters with a default cannot use \
+                    "generic parameters with a default cannot use \
                                                 forward declared identifiers"
                 );
                 err.span_label(
                     span,
-                    "defaulted type parameters cannot be forward declared".to_string(),
+                    "defaulted generic parameters cannot be forward declared".to_string(),
                 );
                 err
             }
@@ -606,7 +606,7 @@ impl<'a> Resolver<'a> {
     /// Lookup typo candidate in scope for a macro or import.
     fn early_lookup_typo_candidate(
         &mut self,
-        scope_set: ScopeSet,
+        scope_set: ScopeSet<'a>,
         parent_scope: &ParentScope<'a>,
         ident: Ident,
         filter_fn: &impl Fn(Res) -> bool,
@@ -662,7 +662,7 @@ impl<'a> Resolver<'a> {
                     let root_module = this.resolve_crate_root(root_ident);
                     this.add_module_candidates(root_module, &mut suggestions, filter_fn);
                 }
-                Scope::Module(module) => {
+                Scope::Module(module, _) => {
                     this.add_module_candidates(module, &mut suggestions, filter_fn);
                 }
                 Scope::RegisteredAttrs => {
@@ -758,17 +758,14 @@ impl<'a> Resolver<'a> {
     {
         let mut candidates = Vec::new();
         let mut seen_modules = FxHashSet::default();
-        let not_local_module = crate_name.name != kw::Crate;
-        let mut worklist =
-            vec![(start_module, Vec::<ast::PathSegment>::new(), true, not_local_module)];
+        let mut worklist = vec![(start_module, Vec::<ast::PathSegment>::new(), true)];
         let mut worklist_via_import = vec![];
 
-        while let Some((in_module, path_segments, accessible, in_module_is_extern)) =
-            match worklist.pop() {
-                None => worklist_via_import.pop(),
-                Some(x) => Some(x),
-            }
-        {
+        while let Some((in_module, path_segments, accessible)) = match worklist.pop() {
+            None => worklist_via_import.pop(),
+            Some(x) => Some(x),
+        } {
+            let in_module_is_extern = !in_module.def_id().unwrap().is_local();
             // We have to visit module children in deterministic order to avoid
             // instabilities in reported imports (#43552).
             in_module.for_each_child(self, |this, ident, ns, name_binding| {
@@ -850,11 +847,10 @@ impl<'a> Resolver<'a> {
                         name_binding.is_extern_crate() && lookup_ident.span.rust_2018();
 
                     if !is_extern_crate_that_also_appears_in_prelude {
-                        let is_extern = in_module_is_extern || name_binding.is_extern_crate();
                         // add the module to the lookup
                         if seen_modules.insert(module.def_id().unwrap()) {
                             if via_import { &mut worklist_via_import } else { &mut worklist }
-                                .push((module, path_segments, child_accessible, is_extern));
+                                .push((module, path_segments, child_accessible));
                         }
                     }
                 }
diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index bd0296751a5..26858915f45 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -156,6 +156,21 @@ impl<'a> NameResolution<'a> {
     }
 }
 
+// Reexports of the form `pub use foo as bar;` where `foo` is `extern crate foo;`
+// are permitted for backward-compatibility under a deprecation lint.
+fn pub_use_of_private_extern_crate_hack(import: &Import<'_>, binding: &NameBinding<'_>) -> bool {
+    match (&import.kind, &binding.kind) {
+        (
+            ImportKind::Single { .. },
+            NameBindingKind::Import {
+                import: Import { kind: ImportKind::ExternCrate { .. }, .. },
+                ..
+            },
+        ) => import.vis.get() == ty::Visibility::Public,
+        _ => false,
+    }
+}
+
 impl<'a> Resolver<'a> {
     crate fn resolve_ident_in_module_unadjusted(
         &mut self,
@@ -263,10 +278,7 @@ impl<'a> Resolver<'a> {
                     return Err((Determined, Weak::No));
                 }
             }
-            // `extern crate` are always usable for backwards compatibility, see issue #37020,
-            // remove this together with `PUB_USE_OF_PRIVATE_EXTERN_CRATE`.
-            let usable = this.is_accessible_from(binding.vis, parent_scope.module)
-                || binding.is_extern_crate();
+            let usable = this.is_accessible_from(binding.vis, parent_scope.module);
             if usable { Ok(binding) } else { Err((Determined, Weak::No)) }
         };
 
@@ -309,10 +321,7 @@ impl<'a> Resolver<'a> {
                             }
                         }
 
-                        if !(self.is_accessible_from(binding.vis, parent_scope.module) ||
-                       // Remove this together with `PUB_USE_OF_PRIVATE_EXTERN_CRATE`
-                       (self.last_import_segment && binding.is_extern_crate()))
-                        {
+                        if !self.is_accessible_from(binding.vis, parent_scope.module) {
                             self.privacy_errors.push(PrivacyError {
                                 ident,
                                 binding,
@@ -455,9 +464,8 @@ impl<'a> Resolver<'a> {
         binding: &'a NameBinding<'a>,
         import: &'a Import<'a>,
     ) -> &'a NameBinding<'a> {
-        let vis = if binding.vis.is_at_least(import.vis.get(), self) ||
-                     // cf. `PUB_USE_OF_PRIVATE_EXTERN_CRATE`
-                     !import.is_glob() && binding.is_extern_crate()
+        let vis = if binding.vis.is_at_least(import.vis.get(), self)
+            || pub_use_of_private_extern_crate_hack(import, binding)
         {
             import.vis.get()
         } else {
@@ -947,14 +955,14 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                 }
                 return None;
             }
-            PathResult::NonModule(path_res) if path_res.base_res() == Res::Err => {
+            PathResult::NonModule(_) => {
                 if no_ambiguity {
                     assert!(import.imported_module.get().is_none());
                 }
                 // The error was already reported earlier.
                 return None;
             }
-            PathResult::Indeterminate | PathResult::NonModule(..) => unreachable!(),
+            PathResult::Indeterminate => unreachable!(),
         };
 
         let (ident, target, source_bindings, target_bindings, type_ns_only) = match import.kind {
@@ -1188,7 +1196,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         // All namespaces must be re-exported with extra visibility for an error to occur.
         if !any_successful_reexport {
             let (ns, binding) = reexport_error.unwrap();
-            if ns == TypeNS && binding.is_extern_crate() {
+            if pub_use_of_private_extern_crate_hack(import, binding) {
                 let msg = format!(
                     "extern crate `{}` is private, and cannot be \
                                    re-exported (error E0365), consider declaring with \
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 6f24d7e9413..9321f11f659 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -132,10 +132,10 @@ crate enum RibKind<'a> {
     /// We passed through a `macro_rules!` statement
     MacroDefinition(DefId),
 
-    /// All bindings in this rib are type parameters that can't be used
-    /// from the default of a type parameter because they're not declared
-    /// before said type parameter. Also see the `visit_generics` override.
-    ForwardTyParamBanRibKind,
+    /// All bindings in this rib are generic parameters that can't be used
+    /// from the default of a generic parameter because they're not declared
+    /// before said generic parameter. Also see the `visit_generics` override.
+    ForwardGenericParamBanRibKind,
 
     /// We are inside of the type of a const parameter. Can't refer to any
     /// parameters.
@@ -154,7 +154,7 @@ impl RibKind<'_> {
             | ModuleRibKind(_)
             | MacroDefinition(_)
             | ConstParamTyRibKind => false,
-            AssocItemRibKind | ItemRibKind(_) | ForwardTyParamBanRibKind => true,
+            AssocItemRibKind | ItemRibKind(_) | ForwardGenericParamBanRibKind => true,
         }
     }
 }
@@ -555,15 +555,16 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> {
         // provide previous type parameters as they're built. We
         // put all the parameters on the ban list and then remove
         // them one by one as they are processed and become available.
-        let mut default_ban_rib = Rib::new(ForwardTyParamBanRibKind);
+        let mut default_ban_rib = Rib::new(ForwardGenericParamBanRibKind);
         let mut found_default = false;
         default_ban_rib.bindings.extend(generics.params.iter().filter_map(
             |param| match param.kind {
-                GenericParamKind::Const { .. } | GenericParamKind::Lifetime { .. } => None,
-                GenericParamKind::Type { ref default, .. } => {
-                    found_default |= default.is_some();
-                    found_default.then_some((Ident::with_dummy_span(param.ident.name), Res::Err))
+                GenericParamKind::Type { default: Some(_), .. }
+                | GenericParamKind::Const { default: Some(_), .. } => {
+                    found_default = true;
+                    Some((Ident::with_dummy_span(param.ident.name), Res::Err))
                 }
+                _ => None,
             },
         ));
 
@@ -591,8 +592,8 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> {
 
                     if let Some(ref ty) = default {
                         self.ribs[TypeNS].push(default_ban_rib);
-                        self.with_rib(ValueNS, ForwardTyParamBanRibKind, |this| {
-                            // HACK: We use an empty `ForwardTyParamBanRibKind` here which
+                        self.with_rib(ValueNS, ForwardGenericParamBanRibKind, |this| {
+                            // HACK: We use an empty `ForwardGenericParamBanRibKind` here which
                             // is only used to forbid the use of const parameters inside of
                             // type defaults.
                             //
@@ -865,7 +866,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                 | ItemRibKind(..)
                 | ConstantItemRibKind(..)
                 | ModuleRibKind(..)
-                | ForwardTyParamBanRibKind
+                | ForwardGenericParamBanRibKind
                 | ConstParamTyRibKind => {
                     return false;
                 }
@@ -1030,7 +1031,6 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
             }
 
             ItemKind::Static(ref ty, _, ref expr) | ItemKind::Const(_, ref ty, ref expr) => {
-                debug!("resolve_item ItemKind::Const");
                 self.with_item_rib(HasGenericParams::No, |this| {
                     this.visit_ty(ty);
                     if let Some(expr) = expr {
@@ -1596,6 +1596,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                         .try_resolve_as_non_binding(pat_src, pat, bmode, ident, has_sub)
                         .unwrap_or_else(|| self.fresh_binding(ident, pat.id, pat_src, bindings));
                     self.r.record_partial_res(pat.id, PartialRes::new(res));
+                    self.r.record_pat_span(pat.id, pat.span);
                 }
                 PatKind::TupleStruct(ref path, ref sub_patterns) => {
                     self.smart_resolve_path(
@@ -2251,8 +2252,8 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                 visit::walk_expr(self, expr);
             }
 
-            ExprKind::Struct(ref path, ..) => {
-                self.smart_resolve_path(expr.id, None, path, PathSource::Struct);
+            ExprKind::Struct(ref se) => {
+                self.smart_resolve_path(expr.id, None, &se.path, PathSource::Struct);
                 visit::walk_expr(self, expr);
             }
 
@@ -2326,7 +2327,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
 
             ExprKind::Call(ref callee, ref arguments) => {
                 self.resolve_expr(callee, Some(expr));
-                let const_args = self.r.legacy_const_generic_args(callee).unwrap_or(Vec::new());
+                let const_args = self.r.legacy_const_generic_args(callee).unwrap_or_default();
                 for (idx, argument) in arguments.iter().enumerate() {
                     // Constant arguments need to be treated as AnonConst since
                     // that is how they will be later lowered to HIR.
diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs
index 87bf79d722b..e33c374f562 100644
--- a/compiler/rustc_resolve/src/late/diagnostics.rs
+++ b/compiler/rustc_resolve/src/late/diagnostics.rs
@@ -16,11 +16,14 @@ use rustc_hir::def::{self, CtorKind, CtorOf, DefKind};
 use rustc_hir::def_id::{DefId, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::PrimTy;
 use rustc_session::parse::feature_err;
+use rustc_span::edition::Edition;
 use rustc_span::hygiene::MacroKind;
 use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, MultiSpan, Span, DUMMY_SP};
 
+use std::iter;
+
 use tracing::debug;
 
 type Res = def::Res<ast::NodeId>;
@@ -133,7 +136,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         let is_enum_variant = &|res| matches!(res, Res::Def(DefKind::Variant, _));
 
         // Make the base error.
-        let expected = source.descr_expected();
+        let mut expected = source.descr_expected();
         let path_str = Segment::names_to_string(path);
         let item_str = path.last().unwrap().ident;
         let (base_msg, fallback_label, base_span, could_be_expr) = if let Some(res) = res {
@@ -166,6 +169,15 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
             let (mod_prefix, mod_str) = if path.len() == 1 {
                 (String::new(), "this scope".to_string())
             } else if path.len() == 2 && path[0].ident.name == kw::PathRoot {
+                if self.r.session.edition() > Edition::Edition2015 {
+                    // In edition 2018 onwards, the `::foo` syntax may only pull from the extern prelude
+                    // which overrides all other expectations of item type
+                    expected = "crate";
+                    (String::new(), "the list of imported crates".to_string())
+                } else {
+                    (String::new(), "the crate root".to_string())
+                }
+            } else if path.len() == 2 && path[0].ident.name == kw::Crate {
                 (String::new(), "the crate root".to_string())
             } else {
                 let mod_path = &path[..path.len() - 1];
@@ -174,7 +186,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                         PathResult::Module(ModuleOrUniformRoot::Module(module)) => module.res(),
                         _ => None,
                     }
-                    .map_or(String::new(), |res| format!("{} ", res.descr()));
+                    .map_or_else(String::new, |res| format!("{} ", res.descr()));
                 (mod_prefix, format!("`{}`", Segment::names_to_string(mod_path)))
             };
             (
@@ -324,7 +336,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 .lookup_import_candidates(ident, ns, &self.parent_scope, is_enum_variant)
                 .into_iter()
                 .map(|suggestion| import_candidate_to_enum_paths(&suggestion))
-                .filter(|(_, enum_ty_path)| enum_ty_path != "std::prelude::v1")
+                .filter(|(_, enum_ty_path)| !enum_ty_path.starts_with("std::prelude::"))
                 .collect();
             if !enum_candidates.is_empty() {
                 if let (PathSource::Type, Some(span)) =
@@ -444,12 +456,14 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
             }
         }
 
+        let is_macro = base_span.from_expansion() && base_span.desugaring_kind().is_none();
         if !self.type_ascription_suggestion(&mut err, base_span) {
             let mut fallback = false;
             if let (
                 PathSource::Trait(AliasPossibility::Maybe),
                 Some(Res::Def(DefKind::Struct | DefKind::Enum | DefKind::Union, _)),
-            ) = (source, res)
+                false,
+            ) = (source, res, is_macro)
             {
                 if let Some(bounds @ [_, .., _]) = self.diagnostic_metadata.current_trait_object {
                     fallback = true;
@@ -553,6 +567,15 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                         }
                     }
                 }
+            } else if err_code == &rustc_errors::error_code!(E0412) {
+                if let Some(correct) = Self::likely_rust_type(path) {
+                    err.span_suggestion(
+                        span,
+                        "perhaps you intended to use this type",
+                        correct.to_string(),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
             }
         }
 
@@ -907,7 +930,14 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                     let msg = "you might have meant to use `#![feature(trait_alias)]` instead of a \
                                `type` alias";
                     if let Some(span) = self.def_span(def_id) {
-                        err.span_help(span, msg);
+                        if let Ok(snip) = self.r.session.source_map().span_to_snippet(span) {
+                            // The span contains a type alias so we should be able to
+                            // replace `type` with `trait`.
+                            let snip = snip.replacen("type", "trait", 1);
+                            err.span_suggestion(span, msg, snip, Applicability::MaybeIncorrect);
+                        } else {
+                            err.span_help(span, msg);
+                        }
                     } else {
                         err.help(msg);
                     }
@@ -985,9 +1015,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 if let Some(spans) =
                     field_spans.filter(|spans| spans.len() > 0 && fields.len() == spans.len())
                 {
-                    let non_visible_spans: Vec<Span> = fields
-                        .iter()
-                        .zip(spans.iter())
+                    let non_visible_spans: Vec<Span> = iter::zip(&fields, &spans)
                         .filter(|(vis, _)| {
                             !self.r.is_accessible_from(**vis, self.parent_scope.module)
                         })
@@ -1023,10 +1051,10 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 if let Some(span) = self.def_span(def_id) {
                     err.span_label(span, &format!("`{}` defined here", path_str));
                 }
-                let fields =
-                    self.r.field_names.get(&def_id).map_or("/* fields */".to_string(), |fields| {
-                        vec!["_"; fields.len()].join(", ")
-                    });
+                let fields = self.r.field_names.get(&def_id).map_or_else(
+                    || "/* fields */".to_string(),
+                    |fields| vec!["_"; fields.len()].join(", "),
+                );
                 err.span_suggestion(
                     span,
                     "use the tuple variant pattern syntax instead",
@@ -1233,6 +1261,23 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         }
     }
 
+    // Returns the name of the Rust type approximately corresponding to
+    // a type name in another programming language.
+    fn likely_rust_type(path: &[Segment]) -> Option<Symbol> {
+        let name = path[path.len() - 1].ident.as_str();
+        // Common Java types
+        Some(match &*name {
+            "byte" => sym::u8, // In Java, bytes are signed, but in practice one almost always wants unsigned bytes.
+            "short" => sym::i16,
+            "boolean" => sym::bool,
+            "int" => sym::i32,
+            "long" => sym::i64,
+            "float" => sym::f32,
+            "double" => sym::f64,
+            _ => return None,
+        })
+    }
+
     /// Only used in a specific case of type ascription suggestions
     fn get_colon_suggestion_span(&self, start: Span) -> Span {
         let sm = self.r.session.source_map();
diff --git a/compiler/rustc_resolve/src/late/lifetimes.rs b/compiler/rustc_resolve/src/late/lifetimes.rs
index 493f25f4992..174df09cbdb 100644
--- a/compiler/rustc_resolve/src/late/lifetimes.rs
+++ b/compiler/rustc_resolve/src/late/lifetimes.rs
@@ -1,3 +1,4 @@
+// ignore-tidy-filelength
 //! Name resolution for lifetimes.
 //!
 //! Name resolution for lifetimes follows *much* simpler rules than the
@@ -11,7 +12,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{CrateNum, DefIdMap, LOCAL_CRATE};
+use rustc_hir::def_id::DefIdMap;
 use rustc_hir::hir_id::ItemLocalId;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_hir::{GenericArg, GenericParam, LifetimeName, Node, ParamName, QPath};
@@ -26,9 +27,10 @@ use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
 use std::borrow::Cow;
 use std::cell::Cell;
+use std::fmt;
 use std::mem::take;
 
-use tracing::debug;
+use tracing::{debug, span, Level};
 
 // This counts the no of times a lifetime is used
 #[derive(Clone, Copy, Debug)]
@@ -40,9 +42,9 @@ pub enum LifetimeUseSet<'tcx> {
 trait RegionExt {
     fn early(hir_map: &Map<'_>, index: &mut u32, param: &GenericParam<'_>) -> (ParamName, Region);
 
-    fn late(hir_map: &Map<'_>, param: &GenericParam<'_>) -> (ParamName, Region);
+    fn late(index: u32, hir_map: &Map<'_>, param: &GenericParam<'_>) -> (ParamName, Region);
 
-    fn late_anon(index: &Cell<u32>) -> Region;
+    fn late_anon(named_late_bound_vars: u32, index: &Cell<u32>) -> Region;
 
     fn id(&self) -> Option<DefId>;
 
@@ -65,29 +67,32 @@ impl RegionExt for Region {
         (param.name.normalize_to_macros_2_0(), Region::EarlyBound(i, def_id.to_def_id(), origin))
     }
 
-    fn late(hir_map: &Map<'_>, param: &GenericParam<'_>) -> (ParamName, Region) {
+    fn late(idx: u32, hir_map: &Map<'_>, param: &GenericParam<'_>) -> (ParamName, Region) {
         let depth = ty::INNERMOST;
         let def_id = hir_map.local_def_id(param.hir_id);
         let origin = LifetimeDefOrigin::from_param(param);
         debug!(
-            "Region::late: param={:?} depth={:?} def_id={:?} origin={:?}",
-            param, depth, def_id, origin,
+            "Region::late: idx={:?}, param={:?} depth={:?} def_id={:?} origin={:?}",
+            idx, param, depth, def_id, origin,
         );
-        (param.name.normalize_to_macros_2_0(), Region::LateBound(depth, def_id.to_def_id(), origin))
+        (
+            param.name.normalize_to_macros_2_0(),
+            Region::LateBound(depth, idx, def_id.to_def_id(), origin),
+        )
     }
 
-    fn late_anon(index: &Cell<u32>) -> Region {
+    fn late_anon(named_late_bound_vars: u32, index: &Cell<u32>) -> Region {
         let i = index.get();
         index.set(i + 1);
         let depth = ty::INNERMOST;
-        Region::LateBoundAnon(depth, i)
+        Region::LateBoundAnon(depth, named_late_bound_vars + i, i)
     }
 
     fn id(&self) -> Option<DefId> {
         match *self {
             Region::Static | Region::LateBoundAnon(..) => None,
 
-            Region::EarlyBound(_, id, _) | Region::LateBound(_, id, _) | Region::Free(_, id) => {
+            Region::EarlyBound(_, id, _) | Region::LateBound(_, _, id, _) | Region::Free(_, id) => {
                 Some(id)
             }
         }
@@ -95,11 +100,11 @@ impl RegionExt for Region {
 
     fn shifted(self, amount: u32) -> Region {
         match self {
-            Region::LateBound(debruijn, id, origin) => {
-                Region::LateBound(debruijn.shifted_in(amount), id, origin)
+            Region::LateBound(debruijn, idx, id, origin) => {
+                Region::LateBound(debruijn.shifted_in(amount), idx, id, origin)
             }
-            Region::LateBoundAnon(debruijn, index) => {
-                Region::LateBoundAnon(debruijn.shifted_in(amount), index)
+            Region::LateBoundAnon(debruijn, index, anon_index) => {
+                Region::LateBoundAnon(debruijn.shifted_in(amount), index, anon_index)
             }
             _ => self,
         }
@@ -107,11 +112,11 @@ impl RegionExt for Region {
 
     fn shifted_out_to_binder(self, binder: ty::DebruijnIndex) -> Region {
         match self {
-            Region::LateBound(debruijn, id, origin) => {
-                Region::LateBound(debruijn.shifted_out_to_binder(binder), id, origin)
+            Region::LateBound(debruijn, index, id, origin) => {
+                Region::LateBound(debruijn.shifted_out_to_binder(binder), index, id, origin)
             }
-            Region::LateBoundAnon(debruijn, index) => {
-                Region::LateBoundAnon(debruijn.shifted_out_to_binder(binder), index)
+            Region::LateBoundAnon(debruijn, index, anon_index) => {
+                Region::LateBoundAnon(debruijn.shifted_out_to_binder(binder), index, anon_index)
             }
             _ => self,
         }
@@ -135,7 +140,7 @@ impl RegionExt for Region {
 /// FIXME. This struct gets converted to a `ResolveLifetimes` for
 /// actual use. It has the same data, but indexed by `LocalDefId`.  This
 /// is silly.
-#[derive(Default)]
+#[derive(Debug, Default)]
 struct NamedRegionMap {
     // maps from every use of a named (not anonymous) lifetime to a
     // `Region` describing how that region is bound
@@ -146,9 +151,13 @@ struct NamedRegionMap {
     // (b) it DOES appear in the arguments.
     late_bound: HirIdSet,
 
-    // For each type and trait definition, maps type parameters
-    // to the trait object lifetime defaults computed from them.
-    object_lifetime_defaults: HirIdMap<Vec<ObjectLifetimeDefault>>,
+    // Maps relevant hir items to the bound vars on them. These include:
+    // - function defs
+    // - function pointers
+    // - closures
+    // - trait refs
+    // - bound types (like `T` in `for<'a> T<'a>: Foo`)
+    late_bound_vars: HirIdMap<Vec<ty::BoundVariableKind>>,
 }
 
 crate struct LifetimeContext<'a, 'tcx> {
@@ -156,26 +165,16 @@ crate struct LifetimeContext<'a, 'tcx> {
     map: &'a mut NamedRegionMap,
     scope: ScopeRef<'a>,
 
-    /// This is slightly complicated. Our representation for poly-trait-refs contains a single
-    /// binder and thus we only allow a single level of quantification. However,
-    /// the syntax of Rust permits quantification in two places, e.g., `T: for <'a> Foo<'a>`
-    /// and `for <'a, 'b> &'b T: Foo<'a>`. In order to get the De Bruijn indices
-    /// correct when representing these constraints, we should only introduce one
-    /// scope. However, we want to support both locations for the quantifier and
-    /// during lifetime resolution we want precise information (so we can't
-    /// desugar in an earlier phase).
-    ///
-    /// So, if we encounter a quantifier at the outer scope, we set
-    /// `trait_ref_hack` to `true` (and introduce a scope), and then if we encounter
-    /// a quantifier at the inner scope, we error. If `trait_ref_hack` is `false`,
-    /// then we introduce the scope at the inner quantifier.
-    trait_ref_hack: bool,
-
     /// Used to disallow the use of in-band lifetimes in `fn` or `Fn` syntax.
     is_in_fn_syntax: bool,
 
     is_in_const_generic: bool,
 
+    /// Indicates that we only care about the definition of a trait. This should
+    /// be false if the `Item` we are resolving lifetimes for is not a trait or
+    /// we eventually need lifetimes resolve for trait items.
+    trait_definition_only: bool,
+
     /// List of labels in the function/method currently under analysis.
     labels_in_fn: Vec<Ident>,
 
@@ -222,6 +221,14 @@ enum Scope<'a> {
         /// of the resulting opaque type.
         opaque_type_parent: bool,
 
+        scope_type: BinderScopeType,
+
+        /// The late bound vars for a given item are stored by `HirId` to be
+        /// queried later. However, if we enter an elision scope, we have to
+        /// later append the elided bound vars to the list and need to know what
+        /// to append to.
+        hir_id: hir::HirId,
+
         s: ScopeRef<'a>,
     },
 
@@ -249,14 +256,90 @@ enum Scope<'a> {
         s: ScopeRef<'a>,
     },
 
+    /// When we have nested trait refs, we concanetate late bound vars for inner
+    /// trait refs from outer ones. But we also need to include any HRTB
+    /// lifetimes encountered when identifying the trait that an associated type
+    /// is declared on.
+    Supertrait {
+        lifetimes: Vec<ty::BoundVariableKind>,
+        s: ScopeRef<'a>,
+    },
+
+    TraitRefBoundary {
+        s: ScopeRef<'a>,
+    },
+
     Root,
 }
 
+#[derive(Copy, Clone, Debug)]
+enum BinderScopeType {
+    /// Any non-concatenating binder scopes.
+    Normal,
+    /// Within a syntactic trait ref, there may be multiple poly trait refs that
+    /// are nested (under the `associcated_type_bounds` feature). The binders of
+    /// the innner poly trait refs are extended from the outer poly trait refs
+    /// and don't increase the late bound depth. If you had
+    /// `T: for<'a>  Foo<Bar: for<'b> Baz<'a, 'b>>`, then the `for<'b>` scope
+    /// would be `Concatenating`. This also used in trait refs in where clauses
+    /// where we have two binders `for<> T: for<> Foo` (I've intentionally left
+    /// out any lifetimes because they aren't needed to show the two scopes).
+    /// The inner `for<>` has a scope of `Concatenating`.
+    Concatenating,
+}
+
+// A helper struct for debugging scopes without printing parent scopes
+struct TruncatedScopeDebug<'a>(&'a Scope<'a>);
+
+impl<'a> fmt::Debug for TruncatedScopeDebug<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self.0 {
+            Scope::Binder {
+                lifetimes,
+                next_early_index,
+                track_lifetime_uses,
+                opaque_type_parent,
+                scope_type,
+                hir_id,
+                s: _,
+            } => f
+                .debug_struct("Binder")
+                .field("lifetimes", lifetimes)
+                .field("next_early_index", next_early_index)
+                .field("track_lifetime_uses", track_lifetime_uses)
+                .field("opaque_type_parent", opaque_type_parent)
+                .field("scope_type", scope_type)
+                .field("hir_id", hir_id)
+                .field("s", &"..")
+                .finish(),
+            Scope::Body { id, s: _ } => {
+                f.debug_struct("Body").field("id", id).field("s", &"..").finish()
+            }
+            Scope::Elision { elide, s: _ } => {
+                f.debug_struct("Elision").field("elide", elide).field("s", &"..").finish()
+            }
+            Scope::ObjectLifetimeDefault { lifetime, s: _ } => f
+                .debug_struct("ObjectLifetimeDefault")
+                .field("lifetime", lifetime)
+                .field("s", &"..")
+                .finish(),
+            Scope::Supertrait { lifetimes, s: _ } => f
+                .debug_struct("Supertrait")
+                .field("lifetimes", lifetimes)
+                .field("s", &"..")
+                .finish(),
+            Scope::TraitRefBoundary { s: _ } => f.debug_struct("TraitRefBoundary").finish(),
+            Scope::Root => f.debug_struct("Root").finish(),
+        }
+    }
+}
+
 #[derive(Clone, Debug)]
 enum Elide {
     /// Use a fresh anonymous late-bound lifetime each time, by
-    /// incrementing the counter to generate sequential indices.
-    FreshLateAnon(Cell<u32>),
+    /// incrementing the counter to generate sequential indices. All
+    /// anonymous lifetimes must start *after* named bound vars.
+    FreshLateAnon(u32, Cell<u32>),
     /// Always use this one lifetime.
     Exact(Region),
     /// Less or more than one lifetime were found, error on unspecified.
@@ -283,26 +366,94 @@ const ROOT_SCOPE: ScopeRef<'static> = &Scope::Root;
 
 pub fn provide(providers: &mut ty::query::Providers) {
     *providers = ty::query::Providers {
+        resolve_lifetimes_trait_definition,
         resolve_lifetimes,
 
-        named_region_map: |tcx, id| tcx.resolve_lifetimes(LOCAL_CRATE).defs.get(&id),
+        named_region_map: |tcx, id| resolve_lifetimes_for(tcx, id).defs.get(&id),
         is_late_bound_map,
         object_lifetime_defaults_map: |tcx, id| {
-            tcx.resolve_lifetimes(LOCAL_CRATE).object_lifetime_defaults.get(&id)
+            let hir_id = tcx.hir().local_def_id_to_hir_id(id);
+            match tcx.hir().find(hir_id) {
+                Some(Node::Item(item)) => compute_object_lifetime_defaults(tcx, item),
+                _ => None,
+            }
         },
+        late_bound_vars_map: |tcx, id| resolve_lifetimes_for(tcx, id).late_bound_vars.get(&id),
 
         ..*providers
     };
 }
 
-/// Computes the `ResolveLifetimes` map that contains data for the
-/// entire crate. You should not read the result of this query
-/// directly, but rather use `named_region_map`, `is_late_bound_map`,
-/// etc.
-fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> ResolveLifetimes {
-    assert_eq!(for_krate, LOCAL_CRATE);
+/// Like `resolve_lifetimes`, but does not resolve lifetimes for trait items.
+/// Also does not generate any diagnostics.
+///
+/// This is ultimately a subset of the `resolve_lifetimes` work. It effectively
+/// resolves lifetimes only within the trait "header" -- that is, the trait
+/// and supertrait list. In contrast, `resolve_lifetimes` resolves all the
+/// lifetimes within the trait and its items. There is room to refactor this,
+/// for example to resolve lifetimes for each trait item in separate queries,
+/// but it's convenient to do the entire trait at once because the lifetimes
+/// from the trait definition are in scope within the trait items as well.
+///
+/// The reason for this separate call is to resolve what would otherwise
+/// be a cycle. Consider this example:
+///
+/// ```rust
+/// trait Base<'a> {
+///     type BaseItem;
+/// }
+/// trait Sub<'b>: for<'a> Base<'a> {
+///    type SubItem: Sub<BaseItem = &'b u32>;
+/// }
+/// ```
+///
+/// When we resolve `Sub` and all its items, we also have to resolve `Sub<BaseItem = &'b u32>`.
+/// To figure out the index of `'b`, we have to know about the supertraits
+/// of `Sub` so that we can determine that the `for<'a>` will be in scope.
+/// (This is because we -- currently at least -- flatten all the late-bound
+/// lifetimes into a single binder.) This requires us to resolve the
+/// *trait definition* of `Sub`; basically just enough lifetime information
+/// to look at the supertraits.
+#[tracing::instrument(level = "debug", skip(tcx))]
+fn resolve_lifetimes_trait_definition(
+    tcx: TyCtxt<'_>,
+    local_def_id: LocalDefId,
+) -> ResolveLifetimes {
+    do_resolve(tcx, local_def_id, true)
+}
 
-    let named_region_map = krate(tcx);
+/// Computes the `ResolveLifetimes` map that contains data for an entire `Item`.
+/// You should not read the result of this query directly, but rather use
+/// `named_region_map`, `is_late_bound_map`, etc.
+#[tracing::instrument(level = "debug", skip(tcx))]
+fn resolve_lifetimes(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> ResolveLifetimes {
+    do_resolve(tcx, local_def_id, false)
+}
+
+fn do_resolve(
+    tcx: TyCtxt<'_>,
+    local_def_id: LocalDefId,
+    trait_definition_only: bool,
+) -> ResolveLifetimes {
+    let item = tcx.hir().expect_item(tcx.hir().local_def_id_to_hir_id(local_def_id));
+    let mut named_region_map = NamedRegionMap {
+        defs: Default::default(),
+        late_bound: Default::default(),
+        late_bound_vars: Default::default(),
+    };
+    let mut visitor = LifetimeContext {
+        tcx,
+        map: &mut named_region_map,
+        scope: ROOT_SCOPE,
+        is_in_fn_syntax: false,
+        is_in_const_generic: false,
+        trait_definition_only,
+        labels_in_fn: vec![],
+        xcrate_object_lifetime_defaults: Default::default(),
+        lifetime_uses: &mut Default::default(),
+        missing_named_lifetime_spots: vec![],
+    };
+    visitor.visit_item(item);
 
     let mut rl = ResolveLifetimes::default();
 
@@ -314,14 +465,62 @@ fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> ResolveLifetimes {
         let map = rl.late_bound.entry(hir_id.owner).or_default();
         map.insert(hir_id.local_id);
     }
-    for (hir_id, v) in named_region_map.object_lifetime_defaults {
-        let map = rl.object_lifetime_defaults.entry(hir_id.owner).or_default();
+    for (hir_id, v) in named_region_map.late_bound_vars {
+        let map = rl.late_bound_vars.entry(hir_id.owner).or_default();
         map.insert(hir_id.local_id, v);
     }
 
+    debug!(?rl.defs);
     rl
 }
 
+/// Given `any` owner (structs, traits, trait methods, etc.), does lifetime resolution.
+/// There are two important things this does.
+/// First, we have to resolve lifetimes for
+/// the entire *`Item`* that contains this owner, because that's the largest "scope"
+/// where we can have relevant lifetimes.
+/// Second, if we are asking for lifetimes in a trait *definition*, we use `resolve_lifetimes_trait_definition`
+/// instead of `resolve_lifetimes`, which does not descend into the trait items and does not emit diagnostics.
+/// This allows us to avoid cycles. Importantly, if we ask for lifetimes for lifetimes that have an owner
+/// other than the trait itself (like the trait methods or associated types), then we just use the regular
+/// `resolve_lifetimes`.
+fn resolve_lifetimes_for<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx ResolveLifetimes {
+    let item_id = item_for(tcx, def_id);
+    if item_id == def_id {
+        let item = tcx.hir().item(hir::ItemId { def_id: item_id });
+        match item.kind {
+            hir::ItemKind::Trait(..) => tcx.resolve_lifetimes_trait_definition(item_id),
+            _ => tcx.resolve_lifetimes(item_id),
+        }
+    } else {
+        tcx.resolve_lifetimes(item_id)
+    }
+}
+
+/// Finds the `Item` that contains the given `LocalDefId`
+fn item_for(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> LocalDefId {
+    let hir_id = tcx.hir().local_def_id_to_hir_id(local_def_id);
+    match tcx.hir().find(hir_id) {
+        Some(Node::Item(item)) => {
+            return item.def_id;
+        }
+        _ => {}
+    }
+    let item = {
+        let hir = tcx.hir();
+        let mut parent_iter = hir.parent_iter(hir_id);
+        loop {
+            let node = parent_iter.next().map(|n| n.1);
+            match node {
+                Some(hir::Node::Item(item)) => break item.def_id,
+                Some(hir::Node::Crate(_)) | None => bug!("Called `item_for` on an Item."),
+                _ => {}
+            }
+        }
+    };
+    item
+}
+
 fn is_late_bound_map<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: LocalDefId,
@@ -344,37 +543,10 @@ fn is_late_bound_map<'tcx>(
 
             tcx.is_late_bound_map(def_id.expect_local())
         }
-        _ => tcx.resolve_lifetimes(LOCAL_CRATE).late_bound.get(&def_id).map(|lt| (def_id, lt)),
+        _ => resolve_lifetimes_for(tcx, def_id).late_bound.get(&def_id).map(|lt| (def_id, lt)),
     }
 }
 
-fn krate(tcx: TyCtxt<'_>) -> NamedRegionMap {
-    let krate = tcx.hir().krate();
-    let mut map = NamedRegionMap {
-        defs: Default::default(),
-        late_bound: Default::default(),
-        object_lifetime_defaults: compute_object_lifetime_defaults(tcx),
-    };
-    {
-        let mut visitor = LifetimeContext {
-            tcx,
-            map: &mut map,
-            scope: ROOT_SCOPE,
-            trait_ref_hack: false,
-            is_in_fn_syntax: false,
-            is_in_const_generic: false,
-            labels_in_fn: vec![],
-            xcrate_object_lifetime_defaults: Default::default(),
-            lifetime_uses: &mut Default::default(),
-            missing_named_lifetime_spots: vec![],
-        };
-        for item in krate.items.values() {
-            visitor.visit_item(item);
-        }
-    }
-    map
-}
-
 /// In traits, there is an implicit `Self` type parameter which comes before the generics.
 /// We have to account for this when computing the index of the other generic parameters.
 /// This function returns whether there is such an implicit parameter defined on the given item.
@@ -382,6 +554,56 @@ fn sub_items_have_self_param(node: &hir::ItemKind<'_>) -> bool {
     matches!(*node, hir::ItemKind::Trait(..) | hir::ItemKind::TraitAlias(..))
 }
 
+fn late_region_as_bound_region<'tcx>(tcx: TyCtxt<'tcx>, region: &Region) -> ty::BoundVariableKind {
+    match region {
+        Region::LateBound(_, _, def_id, _) => {
+            let name = tcx.hir().name(tcx.hir().local_def_id_to_hir_id(def_id.expect_local()));
+            ty::BoundVariableKind::Region(ty::BrNamed(*def_id, name))
+        }
+        Region::LateBoundAnon(_, _, anon_idx) => {
+            ty::BoundVariableKind::Region(ty::BrAnon(*anon_idx))
+        }
+        _ => bug!("{:?} is not a late region", region),
+    }
+}
+
+impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
+    /// Returns the binders in scope and the type of `Binder` that should be created for a poly trait ref.
+    fn poly_trait_ref_binder_info(&mut self) -> (Vec<ty::BoundVariableKind>, BinderScopeType) {
+        let mut scope = self.scope;
+        let mut supertrait_lifetimes = vec![];
+        loop {
+            match scope {
+                Scope::Body { .. } | Scope::Root => {
+                    break (vec![], BinderScopeType::Normal);
+                }
+
+                Scope::Elision { s, .. } | Scope::ObjectLifetimeDefault { s, .. } => {
+                    scope = s;
+                }
+
+                Scope::Supertrait { s, lifetimes } => {
+                    supertrait_lifetimes = lifetimes.clone();
+                    scope = s;
+                }
+
+                Scope::TraitRefBoundary { .. } => {
+                    // We should only see super trait lifetimes if there is a `Binder` above
+                    assert!(supertrait_lifetimes.is_empty());
+                    break (vec![], BinderScopeType::Normal);
+                }
+
+                Scope::Binder { hir_id, .. } => {
+                    // Nested poly trait refs have the binders concatenated
+                    let mut full_binders =
+                        self.map.late_bound_vars.entry(*hir_id).or_default().clone();
+                    full_binders.extend(supertrait_lifetimes.into_iter());
+                    break (full_binders, BinderScopeType::Concatenating);
+                }
+            }
+        }
+    }
+}
 impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
     type Map = Map<'tcx>;
 
@@ -392,6 +614,12 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
     // We want to nest trait/impl items in their parent, but nothing else.
     fn visit_nested_item(&mut self, _: hir::ItemId) {}
 
+    fn visit_trait_item_ref(&mut self, ii: &'tcx hir::TraitItemRef) {
+        if !self.trait_definition_only {
+            intravisit::walk_trait_item_ref(self, ii)
+        }
+    }
+
     fn visit_nested_body(&mut self, body: hir::BodyId) {
         // Each body has their own set of labels, save labels.
         let saved = take(&mut self.labels_in_fn);
@@ -403,11 +631,58 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
         self.labels_in_fn = saved;
     }
 
+    fn visit_fn(
+        &mut self,
+        fk: intravisit::FnKind<'tcx>,
+        fd: &'tcx hir::FnDecl<'tcx>,
+        b: hir::BodyId,
+        s: rustc_span::Span,
+        hir_id: hir::HirId,
+    ) {
+        let name = match fk {
+            intravisit::FnKind::ItemFn(id, _, _, _) => id.as_str(),
+            intravisit::FnKind::Method(id, _, _) => id.as_str(),
+            intravisit::FnKind::Closure => Symbol::intern("closure").as_str(),
+        };
+        let name: &str = &name;
+        let span = span!(Level::DEBUG, "visit_fn", name);
+        let _enter = span.enter();
+        match fk {
+            // Any `Binders` are handled elsewhere
+            intravisit::FnKind::ItemFn(..) | intravisit::FnKind::Method(..) => {
+                intravisit::walk_fn(self, fk, fd, b, s, hir_id)
+            }
+            intravisit::FnKind::Closure => {
+                self.map.late_bound_vars.insert(hir_id, vec![]);
+                let scope = Scope::Binder {
+                    hir_id,
+                    lifetimes: FxHashMap::default(),
+                    next_early_index: self.next_early_index(),
+                    s: self.scope,
+                    track_lifetime_uses: true,
+                    opaque_type_parent: false,
+                    scope_type: BinderScopeType::Normal,
+                };
+                self.with(scope, move |_old_scope, this| {
+                    intravisit::walk_fn(this, fk, fd, b, s, hir_id)
+                });
+            }
+        }
+    }
+
     fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) {
+        match &item.kind {
+            hir::ItemKind::Impl(hir::Impl { of_trait, .. }) => {
+                if let Some(of_trait) = of_trait {
+                    self.map.late_bound_vars.insert(of_trait.hir_ref_id, Vec::default());
+                }
+            }
+            _ => {}
+        }
         match item.kind {
             hir::ItemKind::Fn(ref sig, ref generics, _) => {
                 self.missing_named_lifetime_spots.push(generics.into());
-                self.visit_early_late(None, &sig.decl, generics, |this| {
+                self.visit_early_late(None, item.hir_id(), &sig.decl, generics, |this| {
                     intravisit::walk_item(this, item);
                 });
                 self.missing_named_lifetime_spots.pop();
@@ -430,6 +705,47 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 // Opaque types are visited when we visit the
                 // `TyKind::OpaqueDef`, so that they have the lifetimes from
                 // their parent opaque_ty in scope.
+                //
+                // The core idea here is that since OpaqueTys are generated with the impl Trait as
+                // their owner, we can keep going until we find the Item that owns that. We then
+                // conservatively add all resolved lifetimes. Otherwise we run into problems in
+                // cases like `type Foo<'a> = impl Bar<As = impl Baz + 'a>`.
+                for (_hir_id, node) in
+                    self.tcx.hir().parent_iter(self.tcx.hir().local_def_id_to_hir_id(item.def_id))
+                {
+                    match node {
+                        hir::Node::Item(parent_item) => {
+                            let resolved_lifetimes: &ResolveLifetimes =
+                                self.tcx.resolve_lifetimes(item_for(self.tcx, parent_item.def_id));
+                            // We need to add *all* deps, since opaque tys may want them from *us*
+                            for (&owner, defs) in resolved_lifetimes.defs.iter() {
+                                defs.iter().for_each(|(&local_id, region)| {
+                                    self.map
+                                        .defs
+                                        .insert(hir::HirId { owner, local_id }, region.clone());
+                                });
+                            }
+                            for (&owner, late_bound) in resolved_lifetimes.late_bound.iter() {
+                                late_bound.iter().for_each(|&local_id| {
+                                    self.map.late_bound.insert(hir::HirId { owner, local_id });
+                                });
+                            }
+                            for (&owner, late_bound_vars) in
+                                resolved_lifetimes.late_bound_vars.iter()
+                            {
+                                late_bound_vars.iter().for_each(|(&local_id, late_bound_vars)| {
+                                    self.map.late_bound_vars.insert(
+                                        hir::HirId { owner, local_id },
+                                        late_bound_vars.clone(),
+                                    );
+                                });
+                            }
+                            break;
+                        }
+                        hir::Node::Crate(_) => bug!("No Item about an OpaqueTy"),
+                        _ => {}
+                    }
+                }
             }
             hir::ItemKind::TyAlias(_, ref generics)
             | hir::ItemKind::Enum(_, ref generics)
@@ -463,16 +779,22 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                         }
                     })
                     .collect();
+                self.map.late_bound_vars.insert(item.hir_id(), vec![]);
                 let scope = Scope::Binder {
+                    hir_id: item.hir_id(),
                     lifetimes,
                     next_early_index: index + non_lifetime_count,
                     opaque_type_parent: true,
                     track_lifetime_uses,
+                    scope_type: BinderScopeType::Normal,
                     s: ROOT_SCOPE,
                 };
                 self.with(scope, |old_scope, this| {
                     this.check_lifetime_params(old_scope, &generics.params);
-                    intravisit::walk_item(this, item);
+                    let scope = Scope::TraitRefBoundary { s: this.scope };
+                    this.with(scope, |_, this| {
+                        intravisit::walk_item(this, item);
+                    });
                 });
                 self.missing_named_lifetime_spots.pop();
             }
@@ -482,7 +804,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
     fn visit_foreign_item(&mut self, item: &'tcx hir::ForeignItem<'tcx>) {
         match item.kind {
             hir::ForeignItemKind::Fn(ref decl, _, ref generics) => {
-                self.visit_early_late(None, decl, generics, |this| {
+                self.visit_early_late(None, item.hir_id(), decl, generics, |this| {
                     intravisit::walk_foreign_item(this, item);
                 })
             }
@@ -495,9 +817,8 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
         }
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     fn visit_ty(&mut self, ty: &'tcx hir::Ty<'tcx>) {
-        debug!("visit_ty: id={:?} ty={:?}", ty.hir_id, ty);
-        debug!("visit_ty: ty.kind={:?}", ty.kind);
         match ty.kind {
             hir::TyKind::BareFn(ref c) => {
                 let next_early_index = self.next_early_index();
@@ -515,21 +836,29 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 };
                 self.missing_named_lifetime_spots
                     .push(MissingLifetimeSpot::HigherRanked { span, span_type });
+                let (lifetimes, binders): (FxHashMap<hir::ParamName, Region>, Vec<_>) = c
+                    .generic_params
+                    .iter()
+                    .filter_map(|param| match param.kind {
+                        GenericParamKind::Lifetime { .. } => Some(param),
+                        _ => None,
+                    })
+                    .enumerate()
+                    .map(|(late_bound_idx, param)| {
+                        let pair = Region::late(late_bound_idx as u32, &self.tcx.hir(), param);
+                        let r = late_region_as_bound_region(self.tcx, &pair.1);
+                        (pair, r)
+                    })
+                    .unzip();
+                self.map.late_bound_vars.insert(ty.hir_id, binders);
                 let scope = Scope::Binder {
-                    lifetimes: c
-                        .generic_params
-                        .iter()
-                        .filter_map(|param| match param.kind {
-                            GenericParamKind::Lifetime { .. } => {
-                                Some(Region::late(&self.tcx.hir(), param))
-                            }
-                            _ => None,
-                        })
-                        .collect(),
+                    hir_id: ty.hir_id,
+                    lifetimes,
                     s: self.scope,
                     next_early_index,
                     track_lifetime_uses: true,
                     opaque_type_parent: false,
+                    scope_type: BinderScopeType::Normal,
                 };
                 self.with(scope, |old_scope, this| {
                     // a bare fn has no bounds, so everything
@@ -540,11 +869,14 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 self.missing_named_lifetime_spots.pop();
                 self.is_in_fn_syntax = was_in_fn_syntax;
             }
-            hir::TyKind::TraitObject(bounds, ref lifetime) => {
-                debug!("visit_ty: TraitObject(bounds={:?}, lifetime={:?})", bounds, lifetime);
-                for bound in bounds {
-                    self.visit_poly_trait_ref(bound, hir::TraitBoundModifier::None);
-                }
+            hir::TyKind::TraitObject(bounds, ref lifetime, _) => {
+                debug!(?bounds, ?lifetime, "TraitObject");
+                let scope = Scope::TraitRefBoundary { s: self.scope };
+                self.with(scope, |_, this| {
+                    for bound in bounds {
+                        this.visit_poly_trait_ref(bound, hir::TraitBoundModifier::None);
+                    }
+                });
                 match lifetime.name {
                     LifetimeName::Implicit => {
                         // For types like `dyn Foo`, we should
@@ -564,7 +896,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                         // resolved the same as the `'_` in `&'_ Foo`.
                         //
                         // cc #48468
-                        self.resolve_elided_lifetimes(vec![lifetime])
+                        self.resolve_elided_lifetimes(&[lifetime])
                     }
                     LifetimeName::Param(_) | LifetimeName::Static => {
                         // If the user wrote an explicit name, use that.
@@ -596,9 +928,12 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
 
                         // Elided lifetimes are not allowed in non-return
                         // position impl Trait
-                        let scope = Scope::Elision { elide: Elide::Forbid, s: self.scope };
+                        let scope = Scope::TraitRefBoundary { s: self.scope };
                         self.with(scope, |_, this| {
-                            intravisit::walk_item(this, opaque_ty);
+                            let scope = Scope::Elision { elide: Elide::Forbid, s: this.scope };
+                            this.with(scope, |_, this| {
+                                intravisit::walk_item(this, opaque_ty);
+                            })
                         });
 
                         return;
@@ -627,7 +962,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                         // well-supported at the moment, so this doesn't work.
                         // In the future, this should be fixed and this error should be removed.
                         let def = self.map.defs.get(&lifetime.hir_id).cloned();
-                        if let Some(Region::LateBound(_, def_id, _)) = def {
+                        if let Some(Region::LateBound(_, _, def_id, _)) = def {
                             if let Some(def_id) = def_id.as_local() {
                                 let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
                                 // Ensure that the parent of the def is an item, not HRTB
@@ -652,14 +987,16 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                                 };
 
                                 if !parent_is_item {
-                                    struct_span_err!(
-                                        self.tcx.sess,
-                                        lifetime.span,
-                                        E0657,
-                                        "`impl Trait` can only capture lifetimes \
-                                             bound at the fn or impl level"
-                                    )
-                                    .emit();
+                                    if !self.trait_definition_only {
+                                        struct_span_err!(
+                                            self.tcx.sess,
+                                            lifetime.span,
+                                            E0657,
+                                            "`impl Trait` can only capture lifetimes \
+                                                bound at the fn or impl level"
+                                        )
+                                        .emit();
+                                    }
                                     self.uninsert_lifetime_on_error(lifetime, def.unwrap());
                                 }
                             }
@@ -670,7 +1007,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 // We want to start our early-bound indices at the end of the parent scope,
                 // not including any parent `impl Trait`s.
                 let mut index = self.next_early_index_for_opaque_type();
-                debug!("visit_ty: index = {}", index);
+                debug!(?index);
 
                 let mut elision = None;
                 let mut lifetimes = FxHashMap::default();
@@ -704,38 +1041,49 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                     }
                 }
                 let next_early_index = index + non_lifetime_count;
+                self.map.late_bound_vars.insert(ty.hir_id, vec![]);
 
                 if let Some(elision_region) = elision {
                     let scope =
                         Scope::Elision { elide: Elide::Exact(elision_region), s: self.scope };
                     self.with(scope, |_old_scope, this| {
                         let scope = Scope::Binder {
+                            hir_id: ty.hir_id,
                             lifetimes,
                             next_early_index,
                             s: this.scope,
                             track_lifetime_uses: true,
                             opaque_type_parent: false,
+                            scope_type: BinderScopeType::Normal,
                         };
                         this.with(scope, |_old_scope, this| {
                             this.visit_generics(generics);
-                            for bound in bounds {
-                                this.visit_param_bound(bound);
-                            }
+                            let scope = Scope::TraitRefBoundary { s: this.scope };
+                            this.with(scope, |_, this| {
+                                for bound in bounds {
+                                    this.visit_param_bound(bound);
+                                }
+                            })
                         });
                     });
                 } else {
                     let scope = Scope::Binder {
+                        hir_id: ty.hir_id,
                         lifetimes,
                         next_early_index,
                         s: self.scope,
                         track_lifetime_uses: true,
                         opaque_type_parent: false,
+                        scope_type: BinderScopeType::Normal,
                     };
                     self.with(scope, |_old_scope, this| {
-                        this.visit_generics(generics);
-                        for bound in bounds {
-                            this.visit_param_bound(bound);
-                        }
+                        let scope = Scope::TraitRefBoundary { s: this.scope };
+                        this.with(scope, |_, this| {
+                            this.visit_generics(generics);
+                            for bound in bounds {
+                                this.visit_param_bound(bound);
+                            }
+                        })
                     });
                 }
             }
@@ -751,6 +1099,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 let tcx = self.tcx;
                 self.visit_early_late(
                     Some(tcx.hir().get_parent_item(trait_item.hir_id())),
+                    trait_item.hir_id(),
                     &sig.decl,
                     &trait_item.generics,
                     |this| intravisit::walk_trait_item(this, trait_item),
@@ -776,22 +1125,28 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                         }
                     })
                     .collect();
+                self.map.late_bound_vars.insert(trait_item.hir_id(), vec![]);
                 let scope = Scope::Binder {
+                    hir_id: trait_item.hir_id(),
                     lifetimes,
                     next_early_index: index + non_lifetime_count,
                     s: self.scope,
                     track_lifetime_uses: true,
                     opaque_type_parent: true,
+                    scope_type: BinderScopeType::Normal,
                 };
                 self.with(scope, |old_scope, this| {
                     this.check_lifetime_params(old_scope, &generics.params);
-                    this.visit_generics(generics);
-                    for bound in bounds {
-                        this.visit_param_bound(bound);
-                    }
-                    if let Some(ty) = ty {
-                        this.visit_ty(ty);
-                    }
+                    let scope = Scope::TraitRefBoundary { s: this.scope };
+                    this.with(scope, |_, this| {
+                        this.visit_generics(generics);
+                        for bound in bounds {
+                            this.visit_param_bound(bound);
+                        }
+                        if let Some(ty) = ty {
+                            this.visit_ty(ty);
+                        }
+                    })
                 });
                 self.missing_named_lifetime_spots.pop();
             }
@@ -813,6 +1168,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 let tcx = self.tcx;
                 self.visit_early_late(
                     Some(tcx.hir().get_parent_item(impl_item.hir_id())),
+                    impl_item.hir_id(),
                     &sig.decl,
                     &impl_item.generics,
                     |this| intravisit::walk_impl_item(this, impl_item),
@@ -825,7 +1181,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 let mut index = self.next_early_index();
                 let mut non_lifetime_count = 0;
                 debug!("visit_ty: index = {}", index);
-                let lifetimes = generics
+                let lifetimes: FxHashMap<hir::ParamName, Region> = generics
                     .params
                     .iter()
                     .filter_map(|param| match param.kind {
@@ -838,17 +1194,23 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                         }
                     })
                     .collect();
+                self.map.late_bound_vars.insert(ty.hir_id, vec![]);
                 let scope = Scope::Binder {
+                    hir_id: ty.hir_id,
                     lifetimes,
                     next_early_index: index + non_lifetime_count,
                     s: self.scope,
                     track_lifetime_uses: true,
                     opaque_type_parent: true,
+                    scope_type: BinderScopeType::Normal,
                 };
                 self.with(scope, |old_scope, this| {
                     this.check_lifetime_params(old_scope, &generics.params);
-                    this.visit_generics(generics);
-                    this.visit_ty(ty);
+                    let scope = Scope::TraitRefBoundary { s: this.scope };
+                    this.with(scope, |_, this| {
+                        this.visit_generics(generics);
+                        this.visit_ty(ty);
+                    })
                 });
                 self.missing_named_lifetime_spots.pop();
             }
@@ -862,10 +1224,10 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
         }
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     fn visit_lifetime(&mut self, lifetime_ref: &'tcx hir::Lifetime) {
-        debug!("visit_lifetime(lifetime_ref={:?})", lifetime_ref);
         if lifetime_ref.is_elided() {
-            self.resolve_elided_lifetimes(vec![lifetime_ref]);
+            self.resolve_elided_lifetimes(&[lifetime_ref]);
             return;
         }
         if lifetime_ref.is_static() {
@@ -897,93 +1259,113 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
     }
 
     fn visit_generics(&mut self, generics: &'tcx hir::Generics<'tcx>) {
-        check_mixed_explicit_and_in_band_defs(self.tcx, &generics.params);
-        for param in generics.params {
-            match param.kind {
-                GenericParamKind::Lifetime { .. } => {}
-                GenericParamKind::Type { ref default, .. } => {
-                    walk_list!(self, visit_param_bound, param.bounds);
-                    if let Some(ref ty) = default {
-                        self.visit_ty(&ty);
+        if !self.trait_definition_only {
+            check_mixed_explicit_and_in_band_defs(self.tcx, &generics.params);
+        }
+        let scope = Scope::TraitRefBoundary { s: self.scope };
+        self.with(scope, |_, this| {
+            for param in generics.params {
+                match param.kind {
+                    GenericParamKind::Lifetime { .. } => {}
+                    GenericParamKind::Type { ref default, .. } => {
+                        walk_list!(this, visit_param_bound, param.bounds);
+                        if let Some(ref ty) = default {
+                            this.visit_ty(&ty);
+                        }
+                    }
+                    GenericParamKind::Const { ref ty, .. } => {
+                        let was_in_const_generic = this.is_in_const_generic;
+                        this.is_in_const_generic = true;
+                        walk_list!(this, visit_param_bound, param.bounds);
+                        this.visit_ty(&ty);
+                        this.is_in_const_generic = was_in_const_generic;
                     }
                 }
-                GenericParamKind::Const { ref ty, .. } => {
-                    let was_in_const_generic = self.is_in_const_generic;
-                    self.is_in_const_generic = true;
-                    walk_list!(self, visit_param_bound, param.bounds);
-                    self.visit_ty(&ty);
-                    self.is_in_const_generic = was_in_const_generic;
-                }
-            }
-        }
-        for predicate in generics.where_clause.predicates {
-            match predicate {
-                &hir::WherePredicate::BoundPredicate(hir::WhereBoundPredicate {
-                    ref bounded_ty,
-                    bounds,
-                    ref bound_generic_params,
-                    ..
-                }) => {
-                    let lifetimes: FxHashMap<_, _> = bound_generic_params
-                        .iter()
-                        .filter_map(|param| match param.kind {
-                            GenericParamKind::Lifetime { .. } => {
-                                Some(Region::late(&self.tcx.hir(), param))
-                            }
-                            _ => None,
-                        })
-                        .collect();
-                    if !lifetimes.is_empty() {
-                        let next_early_index = self.next_early_index();
+            }
+            for predicate in generics.where_clause.predicates {
+                match predicate {
+                    &hir::WherePredicate::BoundPredicate(hir::WhereBoundPredicate {
+                        ref bounded_ty,
+                        bounds,
+                        ref bound_generic_params,
+                        ..
+                    }) => {
+                        let (lifetimes, binders): (FxHashMap<hir::ParamName, Region>, Vec<_>) =
+                            bound_generic_params
+                                .iter()
+                                .filter_map(|param| match param.kind {
+                                    GenericParamKind::Lifetime { .. } => Some(param),
+                                    _ => None,
+                                })
+                                .enumerate()
+                                .map(|(late_bound_idx, param)| {
+                                    let pair =
+                                        Region::late(late_bound_idx as u32, &this.tcx.hir(), param);
+                                    let r = late_region_as_bound_region(this.tcx, &pair.1);
+                                    (pair, r)
+                                })
+                                .unzip();
+                        this.map.late_bound_vars.insert(bounded_ty.hir_id, binders.clone());
+                        let next_early_index = this.next_early_index();
+                        // Even if there are no lifetimes defined here, we still wrap it in a binder
+                        // scope. If there happens to be a nested poly trait ref (an error), that
+                        // will be `Concatenating` anyways, so we don't have to worry about the depth
+                        // being wrong.
                         let scope = Scope::Binder {
+                            hir_id: bounded_ty.hir_id,
                             lifetimes,
-                            s: self.scope,
+                            s: this.scope,
                             next_early_index,
                             track_lifetime_uses: true,
                             opaque_type_parent: false,
+                            scope_type: BinderScopeType::Normal,
                         };
-                        let result = self.with(scope, |old_scope, this| {
+                        this.with(scope, |old_scope, this| {
                             this.check_lifetime_params(old_scope, &bound_generic_params);
                             this.visit_ty(&bounded_ty);
-                            this.trait_ref_hack = true;
                             walk_list!(this, visit_param_bound, bounds);
-                            this.trait_ref_hack = false;
-                        });
-                        result
-                    } else {
-                        self.visit_ty(&bounded_ty);
-                        walk_list!(self, visit_param_bound, bounds);
+                        })
+                    }
+                    &hir::WherePredicate::RegionPredicate(hir::WhereRegionPredicate {
+                        ref lifetime,
+                        bounds,
+                        ..
+                    }) => {
+                        this.visit_lifetime(lifetime);
+                        walk_list!(this, visit_param_bound, bounds);
+                    }
+                    &hir::WherePredicate::EqPredicate(hir::WhereEqPredicate {
+                        ref lhs_ty,
+                        ref rhs_ty,
+                        ..
+                    }) => {
+                        this.visit_ty(lhs_ty);
+                        this.visit_ty(rhs_ty);
                     }
-                }
-                &hir::WherePredicate::RegionPredicate(hir::WhereRegionPredicate {
-                    ref lifetime,
-                    bounds,
-                    ..
-                }) => {
-                    self.visit_lifetime(lifetime);
-                    walk_list!(self, visit_param_bound, bounds);
-                }
-                &hir::WherePredicate::EqPredicate(hir::WhereEqPredicate {
-                    ref lhs_ty,
-                    ref rhs_ty,
-                    ..
-                }) => {
-                    self.visit_ty(lhs_ty);
-                    self.visit_ty(rhs_ty);
                 }
             }
-        }
+        })
     }
 
     fn visit_param_bound(&mut self, bound: &'tcx hir::GenericBound<'tcx>) {
         match bound {
-            hir::GenericBound::LangItemTrait { .. } if !self.trait_ref_hack => {
+            hir::GenericBound::LangItemTrait(_, _, hir_id, _) => {
+                // FIXME(jackh726): This is pretty weird. `LangItemTrait` doesn't go
+                // through the regular poly trait ref code, so we don't get another
+                // chance to introduce a binder. For now, I'm keeping the existing logic
+                // of "if there isn't a Binder scope above us, add one", but I
+                // imagine there's a better way to go about this.
+                let (binders, scope_type) = self.poly_trait_ref_binder_info();
+
+                self.map.late_bound_vars.insert(*hir_id, binders);
                 let scope = Scope::Binder {
+                    hir_id: *hir_id,
                     lifetimes: FxHashMap::default(),
                     s: self.scope,
                     next_early_index: self.next_early_index(),
                     track_lifetime_uses: true,
                     opaque_type_parent: false,
+                    scope_type,
                 };
                 self.with(scope, |_, this| {
                     intravisit::walk_param_bound(this, bound);
@@ -1002,48 +1384,53 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
 
         let should_pop_missing_lt = self.is_trait_ref_fn_scope(trait_ref);
 
-        let trait_ref_hack = take(&mut self.trait_ref_hack);
-        if !trait_ref_hack
-            || trait_ref
-                .bound_generic_params
-                .iter()
-                .any(|param| matches!(param.kind, GenericParamKind::Lifetime { .. }))
-        {
-            if trait_ref_hack {
-                struct_span_err!(
-                    self.tcx.sess,
-                    trait_ref.span,
-                    E0316,
-                    "nested quantification of lifetimes"
-                )
-                .emit();
-            }
-            let next_early_index = self.next_early_index();
-            let scope = Scope::Binder {
-                lifetimes: trait_ref
-                    .bound_generic_params
-                    .iter()
-                    .filter_map(|param| match param.kind {
-                        GenericParamKind::Lifetime { .. } => {
-                            Some(Region::late(&self.tcx.hir(), param))
-                        }
-                        _ => None,
-                    })
-                    .collect(),
-                s: self.scope,
-                next_early_index,
-                track_lifetime_uses: true,
-                opaque_type_parent: false,
-            };
-            self.with(scope, |old_scope, this| {
-                this.check_lifetime_params(old_scope, &trait_ref.bound_generic_params);
-                walk_list!(this, visit_generic_param, trait_ref.bound_generic_params);
-                this.visit_trait_ref(&trait_ref.trait_ref);
+        let next_early_index = self.next_early_index();
+        let (mut binders, scope_type) = self.poly_trait_ref_binder_info();
+
+        let initial_bound_vars = binders.len() as u32;
+        let mut lifetimes: FxHashMap<hir::ParamName, Region> = FxHashMap::default();
+        let binders_iter = trait_ref
+            .bound_generic_params
+            .iter()
+            .filter_map(|param| match param.kind {
+                GenericParamKind::Lifetime { .. } => Some(param),
+                _ => None,
+            })
+            .enumerate()
+            .map(|(late_bound_idx, param)| {
+                let pair = Region::late(
+                    initial_bound_vars + late_bound_idx as u32,
+                    &self.tcx.hir(),
+                    param,
+                );
+                let r = late_region_as_bound_region(self.tcx, &pair.1);
+                lifetimes.insert(pair.0, pair.1);
+                r
             });
-        } else {
-            self.visit_trait_ref(&trait_ref.trait_ref);
-        }
-        self.trait_ref_hack = trait_ref_hack;
+        binders.extend(binders_iter);
+
+        debug!(?binders);
+        self.map.late_bound_vars.insert(trait_ref.trait_ref.hir_ref_id, binders);
+
+        // Always introduce a scope here, even if this is in a where clause and
+        // we introduced the binders around the bounded Ty. In that case, we
+        // just reuse the concatenation functionality also present in nested trait
+        // refs.
+        let scope = Scope::Binder {
+            hir_id: trait_ref.trait_ref.hir_ref_id,
+            lifetimes,
+            s: self.scope,
+            next_early_index,
+            track_lifetime_uses: true,
+            opaque_type_parent: false,
+            scope_type,
+        };
+        self.with(scope, |old_scope, this| {
+            this.check_lifetime_params(old_scope, &trait_ref.bound_generic_params);
+            walk_list!(this, visit_generic_param, trait_ref.bound_generic_params);
+            this.visit_trait_ref(&trait_ref.trait_ref);
+        });
+
         if should_pop_missing_lt {
             self.missing_named_lifetime_spots.pop();
         }
@@ -1193,7 +1580,9 @@ fn extract_labels(ctxt: &mut LifetimeContext<'_, '_>, body: &hir::Body<'_>) {
             match *scope {
                 Scope::Body { s, .. }
                 | Scope::Elision { s, .. }
-                | Scope::ObjectLifetimeDefault { s, .. } => {
+                | Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. } => {
                     scope = s;
                 }
 
@@ -1224,55 +1613,53 @@ fn extract_labels(ctxt: &mut LifetimeContext<'_, '_>, body: &hir::Body<'_>) {
     }
 }
 
-fn compute_object_lifetime_defaults(tcx: TyCtxt<'_>) -> HirIdMap<Vec<ObjectLifetimeDefault>> {
-    let mut map = HirIdMap::default();
-    for item in tcx.hir().krate().items.values() {
-        match item.kind {
-            hir::ItemKind::Struct(_, ref generics)
-            | hir::ItemKind::Union(_, ref generics)
-            | hir::ItemKind::Enum(_, ref generics)
-            | hir::ItemKind::OpaqueTy(hir::OpaqueTy {
-                ref generics, impl_trait_fn: None, ..
-            })
-            | hir::ItemKind::TyAlias(_, ref generics)
-            | hir::ItemKind::Trait(_, _, ref generics, ..) => {
-                let result = object_lifetime_defaults_for_item(tcx, generics);
-
-                // Debugging aid.
-                if tcx.sess.contains_name(&item.attrs, sym::rustc_object_lifetime_default) {
-                    let object_lifetime_default_reprs: String = result
-                        .iter()
-                        .map(|set| match *set {
-                            Set1::Empty => "BaseDefault".into(),
-                            Set1::One(Region::Static) => "'static".into(),
-                            Set1::One(Region::EarlyBound(mut i, _, _)) => generics
-                                .params
-                                .iter()
-                                .find_map(|param| match param.kind {
-                                    GenericParamKind::Lifetime { .. } => {
-                                        if i == 0 {
-                                            return Some(param.name.ident().to_string().into());
-                                        }
-                                        i -= 1;
-                                        None
+fn compute_object_lifetime_defaults(
+    tcx: TyCtxt<'_>,
+    item: &hir::Item<'_>,
+) -> Option<Vec<ObjectLifetimeDefault>> {
+    match item.kind {
+        hir::ItemKind::Struct(_, ref generics)
+        | hir::ItemKind::Union(_, ref generics)
+        | hir::ItemKind::Enum(_, ref generics)
+        | hir::ItemKind::OpaqueTy(hir::OpaqueTy { ref generics, impl_trait_fn: None, .. })
+        | hir::ItemKind::TyAlias(_, ref generics)
+        | hir::ItemKind::Trait(_, _, ref generics, ..) => {
+            let result = object_lifetime_defaults_for_item(tcx, generics);
+
+            // Debugging aid.
+            let attrs = tcx.hir().attrs(item.hir_id());
+            if tcx.sess.contains_name(attrs, sym::rustc_object_lifetime_default) {
+                let object_lifetime_default_reprs: String = result
+                    .iter()
+                    .map(|set| match *set {
+                        Set1::Empty => "BaseDefault".into(),
+                        Set1::One(Region::Static) => "'static".into(),
+                        Set1::One(Region::EarlyBound(mut i, _, _)) => generics
+                            .params
+                            .iter()
+                            .find_map(|param| match param.kind {
+                                GenericParamKind::Lifetime { .. } => {
+                                    if i == 0 {
+                                        return Some(param.name.ident().to_string().into());
                                     }
-                                    _ => None,
-                                })
-                                .unwrap(),
-                            Set1::One(_) => bug!(),
-                            Set1::Many => "Ambiguous".into(),
-                        })
-                        .collect::<Vec<Cow<'static, str>>>()
-                        .join(",");
-                    tcx.sess.span_err(item.span, &object_lifetime_default_reprs);
-                }
-
-                map.insert(item.hir_id(), result);
+                                    i -= 1;
+                                    None
+                                }
+                                _ => None,
+                            })
+                            .unwrap(),
+                        Set1::One(_) => bug!(),
+                        Set1::Many => "Ambiguous".into(),
+                    })
+                    .collect::<Vec<Cow<'static, str>>>()
+                    .join(",");
+                tcx.sess.span_err(item.span, &object_lifetime_default_reprs);
             }
-            _ => {}
+
+            Some(result)
         }
+        _ => None,
     }
-    map
 }
 
 /// Scan the bounds and where-clauses on parameters to extract bounds
@@ -1388,18 +1775,22 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             tcx: *tcx,
             map,
             scope: &wrap_scope,
-            trait_ref_hack: self.trait_ref_hack,
             is_in_fn_syntax: self.is_in_fn_syntax,
             is_in_const_generic: self.is_in_const_generic,
+            trait_definition_only: self.trait_definition_only,
             labels_in_fn,
             xcrate_object_lifetime_defaults,
             lifetime_uses,
             missing_named_lifetime_spots,
         };
-        debug!("entering scope {:?}", this.scope);
-        f(self.scope, &mut this);
-        this.check_uses_for_lifetimes_defined_by_scope();
-        debug!("exiting scope {:?}", this.scope);
+        let span = tracing::debug_span!("scope", scope = ?TruncatedScopeDebug(&this.scope));
+        {
+            let _enter = span.enter();
+            f(self.scope, &mut this);
+            if !self.trait_definition_only {
+                this.check_uses_for_lifetimes_defined_by_scope();
+            }
+        }
         self.labels_in_fn = this.labels_in_fn;
         self.xcrate_object_lifetime_defaults = this.xcrate_object_lifetime_defaults;
         self.missing_named_lifetime_spots = this.missing_named_lifetime_spots;
@@ -1551,7 +1942,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             .values()
             .flat_map(|region| match region {
                 Region::EarlyBound(_, def_id, _)
-                | Region::LateBound(_, def_id, _)
+                | Region::LateBound(_, _, def_id, _)
                 | Region::Free(_, def_id) => Some(*def_id),
 
                 Region::LateBoundAnon(..) | Region::Static => None,
@@ -1697,6 +2088,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
     fn visit_early_late<F>(
         &mut self,
         parent_id: Option<hir::HirId>,
+        hir_id: hir::HirId,
         decl: &'tcx hir::FnDecl<'tcx>,
         generics: &'tcx hir::Generics<'tcx>,
         walk: F,
@@ -1706,31 +2098,34 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
         insert_late_bound_lifetimes(self.map, decl, generics);
 
         // Find the start of nested early scopes, e.g., in methods.
-        let mut index = 0;
+        let mut next_early_index = 0;
         if let Some(parent_id) = parent_id {
             let parent = self.tcx.hir().expect_item(parent_id);
             if sub_items_have_self_param(&parent.kind) {
-                index += 1; // Self comes before lifetimes
+                next_early_index += 1; // Self comes before lifetimes
             }
             match parent.kind {
                 hir::ItemKind::Trait(_, _, ref generics, ..)
                 | hir::ItemKind::Impl(hir::Impl { ref generics, .. }) => {
-                    index += generics.params.len() as u32;
+                    next_early_index += generics.params.len() as u32;
                 }
                 _ => {}
             }
         }
 
         let mut non_lifetime_count = 0;
-        let lifetimes = generics
+        let mut named_late_bound_vars = 0;
+        let lifetimes: FxHashMap<hir::ParamName, Region> = generics
             .params
             .iter()
             .filter_map(|param| match param.kind {
                 GenericParamKind::Lifetime { .. } => {
                     if self.map.late_bound.contains(&param.hir_id) {
-                        Some(Region::late(&self.tcx.hir(), param))
+                        let late_bound_idx = named_late_bound_vars;
+                        named_late_bound_vars += 1;
+                        Some(Region::late(late_bound_idx, &self.tcx.hir(), param))
                     } else {
-                        Some(Region::early(&self.tcx.hir(), &mut index, param))
+                        Some(Region::early(&self.tcx.hir(), &mut next_early_index, param))
                     }
                 }
                 GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {
@@ -1739,14 +2134,35 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 }
             })
             .collect();
-        let next_early_index = index + non_lifetime_count;
+        let next_early_index = next_early_index + non_lifetime_count;
 
+        let binders: Vec<_> = generics
+            .params
+            .iter()
+            .filter_map(|param| match param.kind {
+                GenericParamKind::Lifetime { .. }
+                    if self.map.late_bound.contains(&param.hir_id) =>
+                {
+                    Some(param)
+                }
+                _ => None,
+            })
+            .enumerate()
+            .map(|(late_bound_idx, param)| {
+                let pair = Region::late(late_bound_idx as u32, &self.tcx.hir(), param);
+                let r = late_region_as_bound_region(self.tcx, &pair.1);
+                r
+            })
+            .collect();
+        self.map.late_bound_vars.insert(hir_id, binders);
         let scope = Scope::Binder {
+            hir_id,
             lifetimes,
             next_early_index,
             s: self.scope,
             opaque_type_parent: true,
             track_lifetime_uses: false,
+            scope_type: BinderScopeType::Normal,
         };
         self.with(scope, move |old_scope, this| {
             this.check_lifetime_params(old_scope, &generics.params);
@@ -1769,7 +2185,9 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 Scope::Binder { s, .. }
                 | Scope::Body { s, .. }
                 | Scope::Elision { s, .. }
-                | Scope::ObjectLifetimeDefault { s, .. } => scope = s,
+                | Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. } => scope = s,
             }
         }
     }
@@ -1818,7 +2236,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     break None;
                 }
 
-                Scope::Binder { ref lifetimes, s, .. } => {
+                Scope::Binder { ref lifetimes, scope_type, s, .. } => {
                     match lifetime_ref.name {
                         LifetimeName::Param(param_name) => {
                             if let Some(&def) = lifetimes.get(&param_name.normalize_to_macros_2_0())
@@ -1828,12 +2246,17 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                         }
                         _ => bug!("expected LifetimeName::Param"),
                     }
-
-                    late_depth += 1;
+                    match scope_type {
+                        BinderScopeType::Normal => late_depth += 1,
+                        BinderScopeType::Concatenating => {}
+                    }
                     scope = s;
                 }
 
-                Scope::Elision { s, .. } | Scope::ObjectLifetimeDefault { s, .. } => {
+                Scope::Elision { s, .. }
+                | Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. } => {
                     scope = s;
                 }
             }
@@ -1858,10 +2281,10 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             }
 
             // Check for fn-syntax conflicts with in-band lifetime definitions
-            if self.is_in_fn_syntax {
+            if !self.trait_definition_only && self.is_in_fn_syntax {
                 match def {
                     Region::EarlyBound(_, _, LifetimeDefOrigin::InBand)
-                    | Region::LateBound(_, _, LifetimeDefOrigin::InBand) => {
+                    | Region::LateBound(_, _, _, LifetimeDefOrigin::InBand) => {
                         struct_span_err!(
                             self.tcx.sess,
                             lifetime_ref.span,
@@ -1882,6 +2305,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     | Region::LateBound(
                         _,
                         _,
+                        _,
                         LifetimeDefOrigin::ExplicitOrElided | LifetimeDefOrigin::Error,
                     )
                     | Region::LateBoundAnon(..)
@@ -1915,7 +2339,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
         }
 
         let mut elide_lifetimes = true;
-        let lifetimes = generic_args
+        let lifetimes: Vec<_> = generic_args
             .args
             .iter()
             .filter_map(|arg| match arg {
@@ -1928,8 +2352,10 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 _ => None,
             })
             .collect();
+        // We short-circuit here if all are elided in order to pluralize
+        // possible errors
         if elide_lifetimes {
-            self.resolve_elided_lifetimes(lifetimes);
+            self.resolve_elided_lifetimes(&lifetimes);
         } else {
             lifetimes.iter().for_each(|lt| self.visit_lifetime(lt));
         }
@@ -1982,7 +2408,9 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
 
                         Scope::Binder { s, .. }
                         | Scope::Elision { s, .. }
-                        | Scope::ObjectLifetimeDefault { s, .. } => {
+                        | Scope::ObjectLifetimeDefault { s, .. }
+                        | Scope::Supertrait { s, .. }
+                        | Scope::TraitRefBoundary { s, .. } => {
                             scope = s;
                         }
                     }
@@ -1990,45 +2418,47 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             };
 
             let map = &self.map;
-            let unsubst = if let Some(def_id) = def_id.as_local() {
+            let set_to_region = |set: &ObjectLifetimeDefault| match *set {
+                Set1::Empty => {
+                    if in_body {
+                        None
+                    } else {
+                        Some(Region::Static)
+                    }
+                }
+                Set1::One(r) => {
+                    let lifetimes = generic_args.args.iter().filter_map(|arg| match arg {
+                        GenericArg::Lifetime(lt) => Some(lt),
+                        _ => None,
+                    });
+                    r.subst(lifetimes, map)
+                }
+                Set1::Many => None,
+            };
+            if let Some(def_id) = def_id.as_local() {
                 let id = self.tcx.hir().local_def_id_to_hir_id(def_id);
-                &map.object_lifetime_defaults[&id]
+                self.tcx.object_lifetime_defaults(id).unwrap().iter().map(set_to_region).collect()
             } else {
                 let tcx = self.tcx;
-                self.xcrate_object_lifetime_defaults.entry(def_id).or_insert_with(|| {
-                    tcx.generics_of(def_id)
-                        .params
-                        .iter()
-                        .filter_map(|param| match param.kind {
-                            GenericParamDefKind::Type { object_lifetime_default, .. } => {
-                                Some(object_lifetime_default)
-                            }
-                            GenericParamDefKind::Lifetime | GenericParamDefKind::Const => None,
-                        })
-                        .collect()
-                })
-            };
-            debug!("visit_segment_args: unsubst={:?}", unsubst);
-            unsubst
-                .iter()
-                .map(|set| match *set {
-                    Set1::Empty => {
-                        if in_body {
-                            None
-                        } else {
-                            Some(Region::Static)
-                        }
-                    }
-                    Set1::One(r) => {
-                        let lifetimes = generic_args.args.iter().filter_map(|arg| match arg {
-                            GenericArg::Lifetime(lt) => Some(lt),
-                            _ => None,
-                        });
-                        r.subst(lifetimes, map)
-                    }
-                    Set1::Many => None,
-                })
-                .collect()
+                self.xcrate_object_lifetime_defaults
+                    .entry(def_id)
+                    .or_insert_with(|| {
+                        tcx.generics_of(def_id)
+                            .params
+                            .iter()
+                            .filter_map(|param| match param.kind {
+                                GenericParamDefKind::Type { object_lifetime_default, .. } => {
+                                    Some(object_lifetime_default)
+                                }
+                                GenericParamDefKind::Lifetime
+                                | GenericParamDefKind::Const { .. } => None,
+                            })
+                            .collect()
+                    })
+                    .iter()
+                    .map(set_to_region)
+                    .collect()
+            }
         });
 
         debug!("visit_segment_args: object_lifetime_defaults={:?}", object_lifetime_defaults);
@@ -2079,23 +2509,139 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
         let has_lifetime_parameter =
             generic_args.args.iter().any(|arg| matches!(arg, GenericArg::Lifetime(_)));
 
-        // Resolve lifetimes found in the type `XX` from `Item = XX` bindings.
-        for b in generic_args.bindings {
+        // Resolve lifetimes found in the bindings, so either in the type `XX` in `Item = XX` or
+        // in the trait ref `YY<...>` in `Item: YY<...>`.
+        for binding in generic_args.bindings {
             let scope = Scope::ObjectLifetimeDefault {
                 lifetime: if has_lifetime_parameter { None } else { Some(Region::Static) },
                 s: self.scope,
             };
-            self.with(scope, |_, this| this.visit_assoc_type_binding(b));
+            if let Some(type_def_id) = type_def_id {
+                let lifetimes = LifetimeContext::supertrait_hrtb_lifetimes(
+                    self.tcx,
+                    type_def_id,
+                    binding.ident,
+                );
+                self.with(scope, |_, this| {
+                    let scope =
+                        Scope::Supertrait { lifetimes: lifetimes.unwrap_or(vec![]), s: this.scope };
+                    this.with(scope, |_, this| this.visit_assoc_type_binding(binding));
+                });
+            } else {
+                self.with(scope, |_, this| this.visit_assoc_type_binding(binding));
+            }
+        }
+    }
+
+    /// Returns all the late-bound vars that come into scope from supertrait HRTBs, based on the
+    /// associated type name and starting trait.
+    /// For example, imagine we have
+    /// ```rust
+    /// trait Foo<'a, 'b> {
+    ///   type As;
+    /// }
+    /// trait Bar<'b>: for<'a> Foo<'a, 'b> {}
+    /// trait Bar: for<'b> Bar<'b> {}
+    /// ```
+    /// In this case, if we wanted to the supertrait HRTB lifetimes for `As` on
+    /// the starting trait `Bar`, we would return `Some(['b, 'a])`.
+    fn supertrait_hrtb_lifetimes(
+        tcx: TyCtxt<'tcx>,
+        def_id: DefId,
+        assoc_name: Ident,
+    ) -> Option<Vec<ty::BoundVariableKind>> {
+        let trait_defines_associated_type_named = |trait_def_id: DefId| {
+            tcx.associated_items(trait_def_id)
+                .find_by_name_and_kind(tcx, assoc_name, ty::AssocKind::Type, trait_def_id)
+                .is_some()
+        };
+
+        use smallvec::{smallvec, SmallVec};
+        let mut stack: SmallVec<[(DefId, SmallVec<[ty::BoundVariableKind; 8]>); 8]> =
+            smallvec![(def_id, smallvec![])];
+        let mut visited: FxHashSet<DefId> = FxHashSet::default();
+        loop {
+            let (def_id, bound_vars) = match stack.pop() {
+                Some(next) => next,
+                None => break None,
+            };
+            // See issue #83753. If someone writes an associated type on a non-trait, just treat it as
+            // there being no supertrait HRTBs.
+            match tcx.def_kind(def_id) {
+                DefKind::Trait | DefKind::TraitAlias | DefKind::Impl => {}
+                _ => break None,
+            }
+
+            if trait_defines_associated_type_named(def_id) {
+                break Some(bound_vars.into_iter().collect());
+            }
+            let predicates =
+                tcx.super_predicates_that_define_assoc_type((def_id, Some(assoc_name)));
+            let obligations = predicates.predicates.iter().filter_map(|&(pred, _)| {
+                let bound_predicate = pred.kind();
+                match bound_predicate.skip_binder() {
+                    ty::PredicateKind::Trait(data, _) => {
+                        // The order here needs to match what we would get from `subst_supertrait`
+                        let pred_bound_vars = bound_predicate.bound_vars();
+                        let mut all_bound_vars = bound_vars.clone();
+                        all_bound_vars.extend(pred_bound_vars.iter());
+                        let super_def_id = data.trait_ref.def_id;
+                        Some((super_def_id, all_bound_vars))
+                    }
+                    _ => None,
+                }
+            });
+
+            let obligations = obligations.filter(|o| visited.insert(o.0));
+            stack.extend(obligations);
         }
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     fn visit_fn_like_elision(
         &mut self,
         inputs: &'tcx [hir::Ty<'tcx>],
         output: Option<&'tcx hir::Ty<'tcx>>,
     ) {
         debug!("visit_fn_like_elision: enter");
-        let arg_scope = Scope::Elision { elide: Elide::FreshLateAnon(Cell::new(0)), s: self.scope };
+        let mut scope = &*self.scope;
+        let hir_id = loop {
+            match scope {
+                Scope::Binder { hir_id, .. } => {
+                    break *hir_id;
+                }
+                Scope::Body { id, .. } => break id.hir_id,
+                Scope::ObjectLifetimeDefault { ref s, .. }
+                | Scope::Elision { ref s, .. }
+                | Scope::Supertrait { ref s, .. }
+                | Scope::TraitRefBoundary { ref s, .. } => {
+                    scope = *s;
+                }
+                Scope::Root => {
+                    // See issue #83907. Just bail out from looking inside.
+                    self.tcx.sess.delay_span_bug(
+                        rustc_span::DUMMY_SP,
+                        "In fn_like_elision without appropriate scope above",
+                    );
+                    return;
+                }
+            }
+        };
+        // While not strictly necessary, we gather anon lifetimes *before* actually
+        // visiting the argument types.
+        let mut gather = GatherAnonLifetimes { anon_count: 0 };
+        for input in inputs {
+            gather.visit_ty(input);
+        }
+        let late_bound_vars = self.map.late_bound_vars.entry(hir_id).or_default();
+        let named_late_bound_vars = late_bound_vars.len() as u32;
+        late_bound_vars.extend(
+            (0..gather.anon_count).map(|var| ty::BoundVariableKind::Region(ty::BrAnon(var))),
+        );
+        let arg_scope = Scope::Elision {
+            elide: Elide::FreshLateAnon(named_late_bound_vars, Cell::new(0)),
+            s: self.scope,
+        };
         self.with(arg_scope, |_, this| {
             for input in inputs {
                 this.visit_ty(input);
@@ -2107,7 +2653,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             None => return,
         };
 
-        debug!("visit_fn_like_elision: determine output");
+        debug!("determine output");
 
         // Figure out if there's a body we can get argument names from,
         // and whether there's a `self` argument (treated specially).
@@ -2273,11 +2819,10 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             Elide::Error(arg_lifetimes)
         };
 
-        debug!("visit_fn_like_elision: elide={:?}", elide);
+        debug!(?elide);
 
         let scope = Scope::Elision { elide, s: self.scope };
         self.with(scope, |_, this| this.visit_ty(output));
-        debug!("visit_fn_like_elision: exit");
 
         struct GatherLifetimes<'a> {
             map: &'a NamedRegionMap,
@@ -2298,7 +2843,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     self.outer_index.shift_in(1);
                 }
                 match ty.kind {
-                    hir::TyKind::TraitObject(bounds, ref lifetime) => {
+                    hir::TyKind::TraitObject(bounds, ref lifetime, _) => {
                         for bound in bounds {
                             self.visit_poly_trait_ref(bound, hir::TraitBoundModifier::None);
                         }
@@ -2351,21 +2896,60 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             fn visit_lifetime(&mut self, lifetime_ref: &hir::Lifetime) {
                 if let Some(&lifetime) = self.map.defs.get(&lifetime_ref.hir_id) {
                     match lifetime {
-                        Region::LateBound(debruijn, _, _) | Region::LateBoundAnon(debruijn, _)
+                        Region::LateBound(debruijn, _, _, _)
+                        | Region::LateBoundAnon(debruijn, _, _)
                             if debruijn < self.outer_index =>
                         {
                             self.have_bound_regions = true;
                         }
                         _ => {
+                            // FIXME(jackh726): nested trait refs?
                             self.lifetimes.insert(lifetime.shifted_out_to_binder(self.outer_index));
                         }
                     }
                 }
             }
         }
+
+        struct GatherAnonLifetimes {
+            anon_count: u32,
+        }
+        impl<'v> Visitor<'v> for GatherAnonLifetimes {
+            type Map = intravisit::ErasedMap<'v>;
+
+            fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> {
+                NestedVisitorMap::None
+            }
+
+            fn visit_ty(&mut self, ty: &hir::Ty<'_>) {
+                // If we enter a `BareFn`, then we enter a *new* binding scope
+                if let hir::TyKind::BareFn(_) = ty.kind {
+                    return;
+                }
+                intravisit::walk_ty(self, ty);
+            }
+
+            fn visit_generic_args(
+                &mut self,
+                path_span: Span,
+                generic_args: &'v hir::GenericArgs<'v>,
+            ) {
+                // parenthesized args enter a new elison scope
+                if generic_args.parenthesized {
+                    return;
+                }
+                intravisit::walk_generic_args(self, path_span, generic_args)
+            }
+
+            fn visit_lifetime(&mut self, lifetime_ref: &hir::Lifetime) {
+                if lifetime_ref.is_elided() {
+                    self.anon_count += 1;
+                }
+            }
+        }
     }
 
-    fn resolve_elided_lifetimes(&mut self, lifetime_refs: Vec<&'tcx hir::Lifetime>) {
+    fn resolve_elided_lifetimes(&mut self, lifetime_refs: &[&'tcx hir::Lifetime]) {
         debug!("resolve_elided_lifetimes(lifetime_refs={:?})", lifetime_refs);
 
         if lifetime_refs.is_empty() {
@@ -2384,7 +2968,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
 
                 Scope::Root => break None,
 
-                Scope::Binder { s, ref lifetimes, .. } => {
+                Scope::Binder { s, ref lifetimes, scope_type, .. } => {
                     // collect named lifetimes for suggestions
                     for name in lifetimes.keys() {
                         if let hir::ParamName::Plain(name) = name {
@@ -2392,15 +2976,20 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                             lifetime_spans.push(name.span);
                         }
                     }
-                    late_depth += 1;
+                    match scope_type {
+                        BinderScopeType::Normal => late_depth += 1,
+                        BinderScopeType::Concatenating => {}
+                    }
                     scope = s;
                 }
 
                 Scope::Elision { ref elide, ref s, .. } => {
                     let lifetime = match *elide {
-                        Elide::FreshLateAnon(ref counter) => {
+                        Elide::FreshLateAnon(named_late_bound_vars, ref counter) => {
                             for lifetime_ref in lifetime_refs {
-                                let lifetime = Region::late_anon(counter).shifted(late_depth);
+                                let lifetime = Region::late_anon(named_late_bound_vars, counter)
+                                    .shifted(late_depth);
+
                                 self.insert_lifetime(lifetime_ref, lifetime);
                             }
                             return;
@@ -2421,7 +3010,8 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                                         scope = s;
                                     }
                                     Scope::ObjectLifetimeDefault { ref s, .. }
-                                    | Scope::Elision { ref s, .. } => {
+                                    | Scope::Elision { ref s, .. }
+                                    | Scope::TraitRefBoundary { ref s, .. } => {
                                         scope = s;
                                     }
                                     _ => break,
@@ -2437,7 +3027,9 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     return;
                 }
 
-                Scope::ObjectLifetimeDefault { s, .. } => {
+                Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. } => {
                     scope = s;
                 }
             }
@@ -2546,8 +3138,11 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
         let mut scope = self.scope;
         let lifetime = loop {
             match *scope {
-                Scope::Binder { s, .. } => {
-                    late_depth += 1;
+                Scope::Binder { s, scope_type, .. } => {
+                    match scope_type {
+                        BinderScopeType::Normal => late_depth += 1,
+                        BinderScopeType::Concatenating => {}
+                    }
                     scope = s;
                 }
 
@@ -2556,6 +3151,10 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 Scope::Body { .. } | Scope::ObjectLifetimeDefault { lifetime: None, .. } => return,
 
                 Scope::ObjectLifetimeDefault { lifetime: Some(l), .. } => break l,
+
+                Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } => {
+                    scope = s;
+                }
             }
         };
         self.insert_lifetime(lifetime_ref, lifetime.shifted(late_depth));
@@ -2679,7 +3278,9 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             match *old_scope {
                 Scope::Body { s, .. }
                 | Scope::Elision { s, .. }
-                | Scope::ObjectLifetimeDefault { s, .. } => {
+                | Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. } => {
                     old_scope = s;
                 }
 
@@ -2725,7 +3326,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 // A lifetime only used in a fn argument could as well
                 // be replaced with `'_`, as that would generate a
                 // fresh name, too.
-                Scope::Elision { elide: Elide::FreshLateAnon(_), .. } => break true,
+                Scope::Elision { elide: Elide::FreshLateAnon(..), .. } => break true,
 
                 // In the return type or other such place, `'_` is not
                 // going to make a fresh name, so we cannot
@@ -2735,17 +3336,18 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     elide: Elide::Exact(_) | Elide::Error(_) | Elide::Forbid, ..
                 } => break false,
 
-                Scope::ObjectLifetimeDefault { s, .. } => scope = s,
+                Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. } => scope = s,
             }
         }
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     fn insert_lifetime(&mut self, lifetime_ref: &'tcx hir::Lifetime, def: Region) {
         debug!(
-            "insert_lifetime: {} resolved to {:?} span={:?}",
-            self.tcx.hir().node_to_string(lifetime_ref.hir_id),
-            def,
-            self.tcx.sess.source_map().span_to_string(lifetime_ref.span)
+            node = ?self.tcx.hir().node_to_string(lifetime_ref.hir_id),
+            span = ?self.tcx.sess.source_map().span_to_string(lifetime_ref.span)
         );
         self.map.defs.insert(lifetime_ref.hir_id, def);
 
@@ -2755,16 +3357,16 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             }
 
             Region::Free(_, def_id)
-            | Region::LateBound(_, def_id, _)
+            | Region::LateBound(_, _, def_id, _)
             | Region::EarlyBound(_, def_id, _) => {
                 // A lifetime declared by the user.
                 let track_lifetime_uses = self.track_lifetime_uses();
-                debug!("insert_lifetime: track_lifetime_uses={}", track_lifetime_uses);
+                debug!(?track_lifetime_uses);
                 if track_lifetime_uses && !self.lifetime_uses.contains_key(&def_id) {
-                    debug!("insert_lifetime: first use of {:?}", def_id);
+                    debug!("first use of {:?}", def_id);
                     self.lifetime_uses.insert(def_id, LifetimeUseSet::One(lifetime_ref));
                 } else {
-                    debug!("insert_lifetime: many uses of {:?}", def_id);
+                    debug!("many uses of {:?}", def_id);
                     self.lifetime_uses.insert(def_id, LifetimeUseSet::Many);
                 }
             }
@@ -2790,13 +3392,12 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
 /// "Constrained" basically means that it appears in any type but
 /// not amongst the inputs to a projection. In other words, `<&'a
 /// T as Trait<''b>>::Foo` does not constrain `'a` or `'b`.
+#[tracing::instrument(level = "debug", skip(map))]
 fn insert_late_bound_lifetimes(
     map: &mut NamedRegionMap,
     decl: &hir::FnDecl<'_>,
     generics: &hir::Generics<'_>,
 ) {
-    debug!("insert_late_bound_lifetimes(decl={:?}, generics={:?})", decl, generics);
-
     let mut constrained_by_input = ConstrainedCollector::default();
     for arg_ty in decl.inputs {
         constrained_by_input.visit_ty(arg_ty);
@@ -2805,7 +3406,7 @@ fn insert_late_bound_lifetimes(
     let mut appears_in_output = AllCollector::default();
     intravisit::walk_fn_ret_ty(&mut appears_in_output, &decl.output);
 
-    debug!("insert_late_bound_lifetimes: constrained_by_input={:?}", constrained_by_input.regions);
+    debug!(?constrained_by_input.regions);
 
     // Walk the lifetimes that appear in where clauses.
     //
@@ -2825,10 +3426,7 @@ fn insert_late_bound_lifetimes(
         }
     }
 
-    debug!(
-        "insert_late_bound_lifetimes: appears_in_where_clause={:?}",
-        appears_in_where_clause.regions
-    );
+    debug!(?appears_in_where_clause.regions);
 
     // Late bound regions are those that:
     // - appear in the inputs
@@ -2855,11 +3453,7 @@ fn insert_late_bound_lifetimes(
             continue;
         }
 
-        debug!(
-            "insert_late_bound_lifetimes: lifetime {:?} with id {:?} is late-bound",
-            param.name.ident(),
-            param.hir_id
-        );
+        debug!("lifetime {:?} with id {:?} is late-bound", param.name.ident(), param.hir_id);
 
         let inserted = map.late_bound.insert(param.hir_id);
         assert!(inserted, "visited lifetime {:?} twice", param.hir_id);
diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs
index 7aee718bfa9..129954381c9 100644
--- a/compiler/rustc_resolve/src/lib.rs
+++ b/compiler/rustc_resolve/src/lib.rs
@@ -6,7 +6,7 @@
 //! Paths in macros, imports, expressions, types, patterns are resolved here.
 //! Label and lifetime names are resolved here as well.
 //!
-//! Type-relative name resolution (methods, fields, associated items) happens in `librustc_typeck`.
+//! Type-relative name resolution (methods, fields, associated items) happens in `rustc_typeck`.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(box_patterns)]
@@ -14,9 +14,11 @@
 #![feature(control_flow_enum)]
 #![feature(crate_visibility_modifier)]
 #![feature(format_args_capture)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
+#![allow(rustdoc::private_intra_doc_links)]
 
 pub use rustc_hir::def::{Namespace, PerNS};
 
@@ -25,7 +27,6 @@ use Determinacy::*;
 use rustc_arena::{DroplessArena, TypedArena};
 use rustc_ast::node_id::NodeMap;
 use rustc_ast::ptr::P;
-use rustc_ast::unwrap_or;
 use rustc_ast::visit::{self, Visitor};
 use rustc_ast::{self as ast, NodeId};
 use rustc_ast::{Crate, CRATE_NODE_ID};
@@ -37,12 +38,12 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_data_structures::ptr_key::PtrKey;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder};
-use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
+use rustc_expand::base::{DeriveResolutions, SyntaxExtension, SyntaxExtensionKind};
 use rustc_hir::def::Namespace::*;
 use rustc_hir::def::{self, CtorOf, DefKind, NonMacroAttrKind, PartialRes};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, CRATE_DEF_INDEX};
 use rustc_hir::definitions::{DefKey, DefPathData, Definitions};
-use rustc_hir::{PrimTy, TraitCandidate};
+use rustc_hir::TraitCandidate;
 use rustc_index::vec::IndexVec;
 use rustc_metadata::creader::{CStore, CrateLoader};
 use rustc_middle::hir::exports::ExportMap;
@@ -108,7 +109,9 @@ enum Scope<'a> {
     DeriveHelpersCompat,
     MacroRules(MacroRulesScopeRef<'a>),
     CrateRoot,
-    Module(Module<'a>),
+    // The node ID is for reporting the `PROC_MACRO_DERIVE_RESOLUTION_FALLBACK`
+    // lint if it should be reported.
+    Module(Module<'a>, Option<NodeId>),
     RegisteredAttrs,
     MacroUsePrelude,
     BuiltinAttrs,
@@ -122,13 +125,17 @@ enum Scope<'a> {
 /// with different restrictions when looking up the resolution.
 /// This enum is currently used only for early resolution (imports and macros),
 /// but not for late resolution yet.
-enum ScopeSet {
+#[derive(Clone, Copy)]
+enum ScopeSet<'a> {
     /// All scopes with the given namespace.
     All(Namespace, /*is_import*/ bool),
     /// Crate root, then extern prelude (used for mixed 2015-2018 mode in macros).
     AbsolutePath(Namespace),
     /// All scopes with macro namespace and the given macro kind restriction.
     Macro(MacroKind),
+    /// All scopes with the given namespace, used for partially performing late resolution.
+    /// The node id enables lints and is used for reporting them.
+    Late(Namespace, Module<'a>, Option<NodeId>),
 }
 
 /// Everything you need to know about a name's location to resolve it.
@@ -156,6 +163,12 @@ impl<'a> ParentScope<'a> {
     }
 }
 
+#[derive(Copy, Debug, Clone)]
+enum ImplTraitContext {
+    Existential,
+    Universal(LocalDefId),
+}
+
 #[derive(Eq)]
 struct BindingError {
     name: Symbol,
@@ -222,7 +235,7 @@ enum ResolutionError<'a> {
     ),
     /// Error E0530: `X` bindings cannot shadow `Y`s.
     BindingShadowsSomethingUnacceptable(&'static str, Symbol, &'a NameBinding<'a>),
-    /// Error E0128: type parameters with a default cannot use forward-declared identifiers.
+    /// Error E0128: generic parameters with a default cannot use forward-declared identifiers.
     ForwardDeclaredTyParam, // FIXME(const_generics_defaults)
     /// ERROR E0770: the type of const parameters must not depend on other generic parameters.
     ParamInTyOfConstParam(Symbol),
@@ -232,7 +245,7 @@ enum ResolutionError<'a> {
     ///
     /// This error is only emitted when using `min_const_generics`.
     ParamInNonTrivialAnonConst { name: Symbol, is_type: bool },
-    /// Error E0735: type parameters with a default cannot use `Self`
+    /// Error E0735: generic parameters with a default cannot use `Self`
     SelfInTyParamDefault,
     /// Error E0767: use of unreachable label
     UnreachableLabel { name: Symbol, definition_span: Span, suggestion: Option<LabelSuggestion> },
@@ -839,6 +852,12 @@ enum BuiltinMacroState {
     AlreadySeen(Span),
 }
 
+struct DeriveData {
+    resolutions: DeriveResolutions,
+    helper_attrs: Vec<(usize, Ident)>,
+    has_derive_copy: bool,
+}
+
 /// The main resolver class.
 ///
 /// This is the visitor that walks the whole crate.
@@ -872,6 +891,10 @@ pub struct Resolver<'a> {
     /// "self-confirming" import resolutions during import validation.
     unusable_binding: Option<&'a NameBinding<'a>>,
 
+    // Spans for local variables found during pattern resolution.
+    // Used for suggestions during error reporting.
+    pat_span_map: NodeMap<Span>,
+
     /// Resolutions for nodes that have a single resolution.
     partial_res_map: NodeMap<PartialRes>,
     /// Resolutions for import nodes, which have multiple resolutions in different namespaces.
@@ -961,8 +984,9 @@ pub struct Resolver<'a> {
     output_macro_rules_scopes: FxHashMap<ExpnId, MacroRulesScopeRef<'a>>,
     /// Helper attributes that are in scope for the given expansion.
     helper_attrs: FxHashMap<ExpnId, Vec<Ident>>,
-    /// Resolutions for paths inside the `#[derive(...)]` attribute with the given `ExpnId`.
-    derive_resolutions: FxHashMap<ExpnId, Vec<(Lrc<SyntaxExtension>, ast::Path)>>,
+    /// Ready or in-progress results of resolving paths inside the `#[derive(...)]` attribute
+    /// with the given `ExpnId`.
+    derive_data: FxHashMap<ExpnId, DeriveData>,
 
     /// Avoid duplicated errors for "name already defined".
     name_already_seen: FxHashMap<Symbol, Span>,
@@ -989,8 +1013,9 @@ pub struct Resolver<'a> {
     /// Indices of unnamed struct or variant fields with unresolved attributes.
     placeholder_field_indices: FxHashMap<NodeId, usize>,
     /// When collecting definitions from an AST fragment produced by a macro invocation `ExpnId`
-    /// we know what parent node that fragment should be attached to thanks to this table.
-    invocation_parents: FxHashMap<ExpnId, LocalDefId>,
+    /// we know what parent node that fragment should be attached to thanks to this table,
+    /// and how the `impl Trait` fragments were introduced.
+    invocation_parents: FxHashMap<ExpnId, (LocalDefId, ImplTraitContext)>,
 
     next_disambiguator: FxHashMap<(LocalDefId, DefPathData), u32>,
     /// Some way to know that we are in a *trait* impl in `visit_assoc_item`.
@@ -1205,7 +1230,7 @@ impl<'a> Resolver<'a> {
         node_id_to_def_id.insert(CRATE_NODE_ID, root);
 
         let mut invocation_parents = FxHashMap::default();
-        invocation_parents.insert(ExpnId::root(), root);
+        invocation_parents.insert(ExpnId::root(), (root, ImplTraitContext::Existential));
 
         let mut extern_prelude: FxHashMap<Ident, ExternPreludeEntry<'_>> = session
             .opts
@@ -1249,6 +1274,7 @@ impl<'a> Resolver<'a> {
             last_import_segment: false,
             unusable_binding: None,
 
+            pat_span_map: Default::default(),
             partial_res_map: Default::default(),
             import_res_map: Default::default(),
             label_res_map: Default::default(),
@@ -1297,7 +1323,7 @@ impl<'a> Resolver<'a> {
             invocation_parent_scopes: Default::default(),
             output_macro_rules_scopes: Default::default(),
             helper_attrs: Default::default(),
-            derive_resolutions: Default::default(),
+            derive_data: Default::default(),
             local_macro_def_scopes: FxHashMap::default(),
             name_already_seen: FxHashMap::default(),
             potentially_unused_imports: Vec::new(),
@@ -1459,7 +1485,7 @@ impl<'a> Resolver<'a> {
 
         self.visit_scopes(ScopeSet::All(TypeNS, false), parent_scope, ctxt, |this, scope, _, _| {
             match scope {
-                Scope::Module(module) => {
+                Scope::Module(module, _) => {
                     this.traits_in_module(module, assoc_item, &mut found_traits);
                 }
                 Scope::StdLibPrelude => {
@@ -1623,7 +1649,7 @@ impl<'a> Resolver<'a> {
     /// If the callback returns `Some` result, we stop visiting scopes and return it.
     fn visit_scopes<T>(
         &mut self,
-        scope_set: ScopeSet,
+        scope_set: ScopeSet<'a>,
         parent_scope: &ParentScope<'a>,
         ctxt: SyntaxContext,
         mut visitor: impl FnMut(
@@ -1679,12 +1705,17 @@ impl<'a> Resolver<'a> {
             ScopeSet::All(ns, _) => (ns, None, false),
             ScopeSet::AbsolutePath(ns) => (ns, None, true),
             ScopeSet::Macro(macro_kind) => (MacroNS, Some(macro_kind), false),
+            ScopeSet::Late(ns, ..) => (ns, None, false),
+        };
+        let module = match scope_set {
+            // Start with the specified module.
+            ScopeSet::Late(_, module, _) => module,
+            // Jump out of trait or enum modules, they do not act as scopes.
+            _ => parent_scope.module.nearest_item_scope(),
         };
-        // Jump out of trait or enum modules, they do not act as scopes.
-        let module = parent_scope.module.nearest_item_scope();
         let mut scope = match ns {
             _ if is_absolute_path => Scope::CrateRoot,
-            TypeNS | ValueNS => Scope::Module(module),
+            TypeNS | ValueNS => Scope::Module(module, None),
             MacroNS => Scope::DeriveHelpers(parent_scope.expansion),
         };
         let mut ctxt = ctxt.normalize_to_macros_2_0();
@@ -1749,7 +1780,7 @@ impl<'a> Resolver<'a> {
                     MacroRulesScope::Invocation(invoc_id) => {
                         Scope::MacroRules(self.invocation_parent_scopes[&invoc_id].macro_rules)
                     }
-                    MacroRulesScope::Empty => Scope::Module(module),
+                    MacroRulesScope::Empty => Scope::Module(module, None),
                 },
                 Scope::CrateRoot => match ns {
                     TypeNS => {
@@ -1758,10 +1789,16 @@ impl<'a> Resolver<'a> {
                     }
                     ValueNS | MacroNS => break,
                 },
-                Scope::Module(module) => {
+                Scope::Module(module, prev_lint_id) => {
                     use_prelude = !module.no_implicit_prelude;
-                    match self.hygienic_lexical_parent(module, &mut ctxt) {
-                        Some(parent_module) => Scope::Module(parent_module),
+                    let derive_fallback_lint_id = match scope_set {
+                        ScopeSet::Late(.., lint_id) => lint_id,
+                        _ => None,
+                    };
+                    match self.hygienic_lexical_parent(module, &mut ctxt, derive_fallback_lint_id) {
+                        Some((parent_module, lint_id)) => {
+                            Scope::Module(parent_module, lint_id.or(prev_lint_id))
+                        }
                         None => {
                             ctxt.adjust(ExpnId::root());
                             match ns {
@@ -1817,6 +1854,7 @@ impl<'a> Resolver<'a> {
         ribs: &[Rib<'a>],
     ) -> Option<LexicalScopeBinding<'a>> {
         assert!(ns == TypeNS || ns == ValueNS);
+        let orig_ident = ident;
         if ident.name == kw::Empty {
             return Some(LexicalScopeBinding::Res(Res::Err));
         }
@@ -1866,135 +1904,48 @@ impl<'a> Resolver<'a> {
                 _ => continue,
             };
 
-            let item = self.resolve_ident_in_module_unadjusted(
-                ModuleOrUniformRoot::Module(module),
-                ident,
-                ns,
-                parent_scope,
-                record_used,
-                path_span,
-            );
-            if let Ok(binding) = item {
-                // The ident resolves to an item.
-                return Some(LexicalScopeBinding::Item(binding));
-            }
-
             match module.kind {
                 ModuleKind::Block(..) => {} // We can see through blocks
                 _ => break,
             }
-        }
 
-        ident = normalized_ident;
-        let mut poisoned = None;
-        loop {
-            let mut span_data = ident.span.data();
-            let opt_module = if let Some(node_id) = record_used_id {
-                self.hygienic_lexical_parent_with_compatibility_fallback(
-                    module,
-                    &mut span_data.ctxt,
-                    node_id,
-                    &mut poisoned,
-                )
-            } else {
-                self.hygienic_lexical_parent(module, &mut span_data.ctxt)
-            };
-            ident.span = span_data.span();
-            module = unwrap_or!(opt_module, break);
-            let adjusted_parent_scope = &ParentScope { module, ..*parent_scope };
-            let result = self.resolve_ident_in_module_unadjusted(
+            let item = self.resolve_ident_in_module_unadjusted(
                 ModuleOrUniformRoot::Module(module),
                 ident,
                 ns,
-                adjusted_parent_scope,
+                parent_scope,
                 record_used,
                 path_span,
             );
-
-            match result {
-                Ok(binding) => {
-                    if let Some(node_id) = poisoned {
-                        self.lint_buffer.buffer_lint_with_diagnostic(
-                            lint::builtin::PROC_MACRO_DERIVE_RESOLUTION_FALLBACK,
-                            node_id,
-                            ident.span,
-                            &format!("cannot find {} `{}` in this scope", ns.descr(), ident),
-                            BuiltinLintDiagnostics::ProcMacroDeriveResolutionFallback(ident.span),
-                        );
-                    }
-                    return Some(LexicalScopeBinding::Item(binding));
-                }
-                Err(Determined) => continue,
-                Err(Undetermined) => {
-                    span_bug!(ident.span, "undetermined resolution during main resolution pass")
-                }
-            }
-        }
-
-        if !module.no_implicit_prelude {
-            ident.span.adjust(ExpnId::root());
-            if ns == TypeNS {
-                if let Some(binding) = self.extern_prelude_get(ident, !record_used) {
-                    return Some(LexicalScopeBinding::Item(binding));
-                }
-                if let Some(ident) = self.registered_tools.get(&ident) {
-                    let binding =
-                        (Res::ToolMod, ty::Visibility::Public, ident.span, ExpnId::root())
-                            .to_name_binding(self.arenas);
-                    return Some(LexicalScopeBinding::Item(binding));
-                }
-            }
-            if let Some(prelude) = self.prelude {
-                if let Ok(binding) = self.resolve_ident_in_module_unadjusted(
-                    ModuleOrUniformRoot::Module(prelude),
-                    ident,
-                    ns,
-                    parent_scope,
-                    false,
-                    path_span,
-                ) {
-                    return Some(LexicalScopeBinding::Item(binding));
-                }
-            }
-        }
-
-        if ns == TypeNS {
-            if let Some(prim_ty) = PrimTy::from_name(ident.name) {
-                let binding =
-                    (Res::PrimTy(prim_ty), ty::Visibility::Public, DUMMY_SP, ExpnId::root())
-                        .to_name_binding(self.arenas);
+            if let Ok(binding) = item {
+                // The ident resolves to an item.
                 return Some(LexicalScopeBinding::Item(binding));
             }
         }
-
-        None
+        self.early_resolve_ident_in_lexical_scope(
+            orig_ident,
+            ScopeSet::Late(ns, module, record_used_id),
+            parent_scope,
+            record_used,
+            record_used,
+            path_span,
+        )
+        .ok()
+        .map(LexicalScopeBinding::Item)
     }
 
     fn hygienic_lexical_parent(
         &mut self,
         module: Module<'a>,
         ctxt: &mut SyntaxContext,
-    ) -> Option<Module<'a>> {
+        derive_fallback_lint_id: Option<NodeId>,
+    ) -> Option<(Module<'a>, Option<NodeId>)> {
         if !module.expansion.outer_expn_is_descendant_of(*ctxt) {
-            return Some(self.macro_def_scope(ctxt.remove_mark()));
+            return Some((self.macro_def_scope(ctxt.remove_mark()), None));
         }
 
         if let ModuleKind::Block(..) = module.kind {
-            return Some(module.parent.unwrap().nearest_item_scope());
-        }
-
-        None
-    }
-
-    fn hygienic_lexical_parent_with_compatibility_fallback(
-        &mut self,
-        module: Module<'a>,
-        ctxt: &mut SyntaxContext,
-        node_id: NodeId,
-        poisoned: &mut Option<NodeId>,
-    ) -> Option<Module<'a>> {
-        if let module @ Some(..) = self.hygienic_lexical_parent(module, ctxt) {
-            return module;
+            return Some((module.parent.unwrap().nearest_item_scope(), None));
         }
 
         // We need to support the next case under a deprecation warning
@@ -2008,20 +1959,21 @@ impl<'a> Resolver<'a> {
         // ---- end
         // ```
         // So we have to fall back to the module's parent during lexical resolution in this case.
-        if let Some(parent) = module.parent {
-            // Inner module is inside the macro, parent module is outside of the macro.
-            if module.expansion != parent.expansion
-                && module.expansion.is_descendant_of(parent.expansion)
-            {
-                // The macro is a proc macro derive
-                if let Some(def_id) = module.expansion.expn_data().macro_def_id {
-                    let ext = self.get_macro_by_def_id(def_id);
-                    if ext.builtin_name.is_none()
-                        && ext.macro_kind() == MacroKind::Derive
-                        && parent.expansion.outer_expn_is_descendant_of(*ctxt)
-                    {
-                        *poisoned = Some(node_id);
-                        return module.parent;
+        if derive_fallback_lint_id.is_some() {
+            if let Some(parent) = module.parent {
+                // Inner module is inside the macro, parent module is outside of the macro.
+                if module.expansion != parent.expansion
+                    && module.expansion.is_descendant_of(parent.expansion)
+                {
+                    // The macro is a proc macro derive
+                    if let Some(def_id) = module.expansion.expn_data().macro_def_id {
+                        let ext = self.get_macro_by_def_id(def_id);
+                        if ext.builtin_name.is_none()
+                            && ext.macro_kind() == MacroKind::Derive
+                            && parent.expansion.outer_expn_is_descendant_of(*ctxt)
+                        {
+                            return Some((parent, derive_fallback_lint_id));
+                        }
                     }
                 }
             }
@@ -2433,8 +2385,10 @@ impl<'a> Resolver<'a> {
                                     Applicability::MaybeIncorrect,
                                 )),
                             )
-                        } else {
+                        } else if self.session.edition() == Edition::Edition2015 {
                             (format!("maybe a missing crate `{}`?", ident), None)
+                        } else {
+                            (format!("could not find `{}` in the crate root", ident), None)
                         }
                     } else if i == 0 {
                         if ident
@@ -2444,16 +2398,74 @@ impl<'a> Resolver<'a> {
                             .next()
                             .map_or(false, |c| c.is_ascii_uppercase())
                         {
-                            (format!("use of undeclared type `{}`", ident), None)
+                            // Check whether the name refers to an item in the value namespace.
+                            let suggestion = if ribs.is_some() {
+                                let match_span = match self.resolve_ident_in_lexical_scope(
+                                    ident,
+                                    ValueNS,
+                                    parent_scope,
+                                    None,
+                                    path_span,
+                                    &ribs.unwrap()[ValueNS],
+                                ) {
+                                    // Name matches a local variable. For example:
+                                    // ```
+                                    // fn f() {
+                                    //     let Foo: &str = "";
+                                    //     println!("{}", Foo::Bar); // Name refers to local
+                                    //                               // variable `Foo`.
+                                    // }
+                                    // ```
+                                    Some(LexicalScopeBinding::Res(Res::Local(id))) => {
+                                        Some(*self.pat_span_map.get(&id).unwrap())
+                                    }
+
+                                    // Name matches item from a local name binding
+                                    // created by `use` declaration. For example:
+                                    // ```
+                                    // pub Foo: &str = "";
+                                    //
+                                    // mod submod {
+                                    //     use super::Foo;
+                                    //     println!("{}", Foo::Bar); // Name refers to local
+                                    //                               // binding `Foo`.
+                                    // }
+                                    // ```
+                                    Some(LexicalScopeBinding::Item(name_binding)) => {
+                                        Some(name_binding.span)
+                                    }
+                                    _ => None,
+                                };
+
+                                if let Some(span) = match_span {
+                                    Some((
+                                        vec![(span, String::from(""))],
+                                        format!("`{}` is defined here, but is not a type", ident),
+                                        Applicability::MaybeIncorrect,
+                                    ))
+                                } else {
+                                    None
+                                }
+                            } else {
+                                None
+                            };
+
+                            (format!("use of undeclared type `{}`", ident), suggestion)
                         } else {
                             (format!("use of undeclared crate or module `{}`", ident), None)
                         }
                     } else {
                         let parent = path[i - 1].ident.name;
-                        let parent = if parent == kw::PathRoot {
-                            "crate root".to_owned()
-                        } else {
-                            format!("`{}`", parent)
+                        let parent = match parent {
+                            // ::foo is mounted at the crate root for 2015, and is the extern
+                            // prelude for 2018+
+                            kw::PathRoot if self.session.edition() > Edition::Edition2015 => {
+                                "the list of imported crates".to_owned()
+                            }
+                            kw::PathRoot | kw::Crate => "the crate root".to_owned(),
+                            _ => {
+                                format!("`{}`", parent)
+                            }
                         };
 
                         let mut msg = format!("could not find `{}` in {}", ident, parent);
@@ -2577,8 +2589,8 @@ impl<'a> Resolver<'a> {
         debug!("validate_res_from_ribs({:?})", res);
         let ribs = &all_ribs[rib_index + 1..];
 
-        // An invalid forward use of a type parameter from a previous default.
-        if let ForwardTyParamBanRibKind = all_ribs[rib_index].kind {
+        // An invalid forward use of a generic parameter from a previous default.
+        if let ForwardGenericParamBanRibKind = all_ribs[rib_index].kind {
             if record_used {
                 let res_error = if rib_ident.name == kw::SelfUpper {
                     ResolutionError::SelfInTyParamDefault
@@ -2602,7 +2614,7 @@ impl<'a> Resolver<'a> {
                         | ClosureOrAsyncRibKind
                         | ModuleRibKind(..)
                         | MacroDefinition(..)
-                        | ForwardTyParamBanRibKind => {
+                        | ForwardGenericParamBanRibKind => {
                             // Nothing to do. Continue.
                         }
                         ItemRibKind(_) | FnItemRibKind | AssocItemRibKind => {
@@ -2674,7 +2686,9 @@ impl<'a> Resolver<'a> {
 
                         // We only forbid constant items if we are inside of type defaults,
                         // for example `struct Foo<T, U = [u8; std::mem::size_of::<T>()]>`
-                        ForwardTyParamBanRibKind => {
+                        ForwardGenericParamBanRibKind => {
+                            // FIXME(const_generic_defaults): we may need to distinguish between
+                            // being in type parameter defaults and const parameter defaults
                             in_ty_param_default = true;
                             continue;
                         }
@@ -2767,7 +2781,9 @@ impl<'a> Resolver<'a> {
 
                         // We only forbid constant items if we are inside of type defaults,
                         // for example `struct Foo<T, U = [u8; std::mem::size_of::<T>()]>`
-                        ForwardTyParamBanRibKind => {
+                        ForwardGenericParamBanRibKind => {
+                            // FIXME(const_generic_defaults): we may need to distinguish between
+                            // being in type parameter defaults and const parameter defaults
                             in_ty_param_default = true;
                             continue;
                         }
@@ -2845,6 +2861,11 @@ impl<'a> Resolver<'a> {
         }
     }
 
+    fn record_pat_span(&mut self, node: NodeId, span: Span) {
+        debug!("(recording pat) recording {:?} for {:?}", node, span);
+        self.pat_span_map.insert(node, span);
+    }
+
     fn is_accessible_from(&self, vis: ty::Visibility, module: Module<'a>) -> bool {
         vis.is_accessible_from(module.nearest_parent_mod, self)
     }
diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs
index f7010ca94bd..10e27f33c29 100644
--- a/compiler/rustc_resolve/src/macros.rs
+++ b/compiler/rustc_resolve/src/macros.rs
@@ -4,9 +4,9 @@
 use crate::imports::ImportResolver;
 use crate::Namespace::*;
 use crate::{AmbiguityError, AmbiguityErrorMisc, AmbiguityKind, BuiltinMacroState, Determinacy};
-use crate::{CrateLint, ParentScope, ResolutionError, Resolver, Scope, ScopeSet, Weak};
+use crate::{CrateLint, DeriveData, ParentScope, ResolutionError, Resolver, Scope, ScopeSet, Weak};
 use crate::{ModuleKind, ModuleOrUniformRoot, NameBinding, PathResult, Segment, ToNameBinding};
-use rustc_ast::{self as ast, NodeId};
+use rustc_ast::{self as ast, Inline, ItemKind, ModKind, NodeId};
 use rustc_ast_lowering::ResolverAstLowering;
 use rustc_ast_pretty::pprust;
 use rustc_attr::StabilityLevel;
@@ -14,16 +14,18 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::ptr_key::PtrKey;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::struct_span_err;
-use rustc_expand::base::{Indeterminate, ResolverExpand, SyntaxExtension, SyntaxExtensionKind};
+use rustc_expand::base::{Annotatable, DeriveResolutions, Indeterminate, ResolverExpand};
+use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
 use rustc_expand::compile_declarative_macro;
-use rustc_expand::expand::{AstFragment, Invocation, InvocationKind};
+use rustc_expand::expand::{AstFragment, Invocation, InvocationKind, SupportsMacroExpansion};
 use rustc_feature::is_builtin_attr_name;
 use rustc_hir::def::{self, DefKind, NonMacroAttrKind};
 use rustc_hir::def_id;
 use rustc_hir::PrimTy;
 use rustc_middle::middle::stability;
 use rustc_middle::ty;
-use rustc_session::lint::builtin::{LEGACY_DERIVE_HELPERS, SOFT_UNSTABLE, UNUSED_MACROS};
+use rustc_session::lint::builtin::{LEGACY_DERIVE_HELPERS, PROC_MACRO_DERIVE_RESOLUTION_FALLBACK};
+use rustc_session::lint::builtin::{SOFT_UNSTABLE, UNUSED_MACROS};
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
@@ -153,6 +155,26 @@ crate fn registered_attrs_and_tools(
     (registered_attrs, registered_tools)
 }
 
+// Some feature gates for inner attributes are reported as lints for backward compatibility.
+fn soft_custom_inner_attributes_gate(path: &ast::Path, invoc: &Invocation) -> bool {
+    match &path.segments[..] {
+        // `#![test]`
+        [seg] if seg.ident.name == sym::test => return true,
+        // `#![rustfmt::skip]` on out-of-line modules
+        [seg1, seg2] if seg1.ident.name == sym::rustfmt && seg2.ident.name == sym::skip => {
+            if let InvocationKind::Attr { item, .. } = &invoc.kind {
+                if let Annotatable::Item(item) = item {
+                    if let ItemKind::Mod(_, ModKind::Loaded(_, Inline::No, _)) = item.kind {
+                        return true;
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+    false
+}
+
 impl<'a> ResolverExpand for Resolver<'a> {
     fn next_node_id(&mut self) -> NodeId {
         self.next_node_id()
@@ -257,16 +279,17 @@ impl<'a> ResolverExpand for Resolver<'a> {
 
         // Derives are not included when `invocations` are collected, so we have to add them here.
         let parent_scope = &ParentScope { derives, ..parent_scope };
-        let require_inert = !invoc.fragment_kind.supports_macro_expansion();
+        let supports_macro_expansion = invoc.fragment_kind.supports_macro_expansion();
         let node_id = self.lint_node_id(eager_expansion_root);
         let (ext, res) = self.smart_resolve_macro_path(
             path,
             kind,
-            require_inert,
+            supports_macro_expansion,
             inner_attr,
             parent_scope,
             node_id,
             force,
+            soft_custom_inner_attributes_gate(path, invoc),
         )?;
 
         let span = invoc.span();
@@ -326,7 +349,7 @@ impl<'a> ResolverExpand for Resolver<'a> {
         // nearest closing item - we should try to return the closest parent of the ExpnId
         self.invocation_parents
             .get(&expn_id)
-            .map_or(ast::CRATE_NODE_ID, |id| self.def_id_to_node_id[*id])
+            .map_or(ast::CRATE_NODE_ID, |id| self.def_id_to_node_id[id.0])
     }
 
     fn has_derive_copy(&self, expn_id: ExpnId) -> bool {
@@ -336,8 +359,8 @@ impl<'a> ResolverExpand for Resolver<'a> {
     fn resolve_derives(
         &mut self,
         expn_id: ExpnId,
-        derives: Vec<ast::Path>,
         force: bool,
+        derive_paths: &dyn Fn() -> DeriveResolutions,
     ) -> Result<(), Indeterminate> {
         // Block expansion of the container until we resolve all derives in it.
         // This is required for two reasons:
@@ -345,49 +368,63 @@ impl<'a> ResolverExpand for Resolver<'a> {
         //   is applied, so they have to be produced by the container's expansion rather
         //   than by individual derives.
         // - Derives in the container need to know whether one of them is a built-in `Copy`.
-        // FIXME: Try to cache intermediate results to avoid resolving same derives multiple times.
+        // Temporarily take the data to avoid borrow checker conflicts.
+        let mut derive_data = mem::take(&mut self.derive_data);
+        let entry = derive_data.entry(expn_id).or_insert_with(|| DeriveData {
+            resolutions: derive_paths(),
+            helper_attrs: Vec::new(),
+            has_derive_copy: false,
+        });
         let parent_scope = self.invocation_parent_scopes[&expn_id];
-        let mut exts = Vec::new();
-        let mut helper_attrs = Vec::new();
-        let mut has_derive_copy = false;
-        for path in derives {
-            exts.push((
-                match self.resolve_macro_path(
-                    &path,
-                    Some(MacroKind::Derive),
-                    &parent_scope,
-                    true,
-                    force,
-                ) {
-                    Ok((Some(ext), _)) => {
-                        let span =
-                            path.segments.last().unwrap().ident.span.normalize_to_macros_2_0();
-                        helper_attrs
-                            .extend(ext.helper_attrs.iter().map(|name| Ident::new(*name, span)));
-                        has_derive_copy |= ext.builtin_name == Some(sym::Copy);
-                        ext
-                    }
-                    Ok(_) | Err(Determinacy::Determined) => self.dummy_ext(MacroKind::Derive),
-                    Err(Determinacy::Undetermined) => return Err(Indeterminate),
-                },
-                path,
-            ))
+        for (i, (path, opt_ext)) in entry.resolutions.iter_mut().enumerate() {
+            if opt_ext.is_none() {
+                *opt_ext = Some(
+                    match self.resolve_macro_path(
+                        &path,
+                        Some(MacroKind::Derive),
+                        &parent_scope,
+                        true,
+                        force,
+                    ) {
+                        Ok((Some(ext), _)) => {
+                            if !ext.helper_attrs.is_empty() {
+                                let last_seg = path.segments.last().unwrap();
+                                let span = last_seg.ident.span.normalize_to_macros_2_0();
+                                entry.helper_attrs.extend(
+                                    ext.helper_attrs
+                                        .iter()
+                                        .map(|name| (i, Ident::new(*name, span))),
+                                );
+                            }
+                            entry.has_derive_copy |= ext.builtin_name == Some(sym::Copy);
+                            ext
+                        }
+                        Ok(_) | Err(Determinacy::Determined) => self.dummy_ext(MacroKind::Derive),
+                        Err(Determinacy::Undetermined) => {
+                            assert!(self.derive_data.is_empty());
+                            self.derive_data = derive_data;
+                            return Err(Indeterminate);
+                        }
+                    },
+                );
+            }
         }
-        self.derive_resolutions.insert(expn_id, exts);
-        self.helper_attrs.insert(expn_id, helper_attrs);
+        // Sort helpers in a stable way independent from the derive resolution order.
+        entry.helper_attrs.sort_by_key(|(i, _)| *i);
+        self.helper_attrs
+            .insert(expn_id, entry.helper_attrs.iter().map(|(_, ident)| *ident).collect());
         // Mark this derive as having `Copy` either if it has `Copy` itself or if its parent derive
         // has `Copy`, to support cases like `#[derive(Clone, Copy)] #[derive(Debug)]`.
-        if has_derive_copy || self.has_derive_copy(parent_scope.expansion) {
+        if entry.has_derive_copy || self.has_derive_copy(parent_scope.expansion) {
             self.containers_deriving_copy.insert(expn_id);
         }
+        assert!(self.derive_data.is_empty());
+        self.derive_data = derive_data;
         Ok(())
     }
 
-    fn take_derive_resolutions(
-        &mut self,
-        expn_id: ExpnId,
-    ) -> Option<Vec<(Lrc<SyntaxExtension>, ast::Path)>> {
-        self.derive_resolutions.remove(&expn_id)
+    fn take_derive_resolutions(&mut self, expn_id: ExpnId) -> Option<DeriveResolutions> {
+        self.derive_data.remove(&expn_id).map(|data| data.resolutions)
     }
 
     // The function that implements the resolution logic of `#[cfg_accessible(path)]`.
@@ -435,11 +472,12 @@ impl<'a> Resolver<'a> {
         &mut self,
         path: &ast::Path,
         kind: MacroKind,
-        require_inert: bool,
+        supports_macro_expansion: SupportsMacroExpansion,
         inner_attr: bool,
         parent_scope: &ParentScope<'a>,
         node_id: NodeId,
         force: bool,
+        soft_custom_inner_attributes_gate: bool,
     ) -> Result<(Lrc<SyntaxExtension>, Res), Indeterminate> {
         let (ext, res) = match self.resolve_macro_path(path, Some(kind), parent_scope, true, force)
         {
@@ -482,8 +520,17 @@ impl<'a> Resolver<'a> {
 
         let unexpected_res = if ext.macro_kind() != kind {
             Some((kind.article(), kind.descr_expected()))
-        } else if require_inert && matches!(res, Res::Def(..)) {
-            Some(("a", "non-macro attribute"))
+        } else if matches!(res, Res::Def(..)) {
+            match supports_macro_expansion {
+                SupportsMacroExpansion::No => Some(("a", "non-macro attribute")),
+                SupportsMacroExpansion::Yes { supports_inner_attrs } => {
+                    if inner_attr && !supports_inner_attrs {
+                        Some(("a", "non-macro inner attribute"))
+                    } else {
+                        None
+                    }
+                }
+            }
         } else {
             None
         };
@@ -507,7 +554,7 @@ impl<'a> Resolver<'a> {
                 Res::NonMacroAttr(..) => "custom inner attributes are unstable",
                 _ => unreachable!(),
             };
-            if path == &sym::test {
+            if soft_custom_inner_attributes_gate {
                 self.session.parse_sess.buffer_lint(SOFT_UNSTABLE, path.span, node_id, msg);
             } else {
                 feature_err(&self.session.parse_sess, sym::custom_inner_attributes, path.span, msg)
@@ -610,7 +657,7 @@ impl<'a> Resolver<'a> {
     crate fn early_resolve_ident_in_lexical_scope(
         &mut self,
         orig_ident: Ident,
-        scope_set: ScopeSet,
+        scope_set: ScopeSet<'a>,
         parent_scope: &ParentScope<'a>,
         record_used: bool,
         force: bool,
@@ -637,6 +684,7 @@ impl<'a> Resolver<'a> {
             ScopeSet::All(ns, is_import) => (ns, None, is_import),
             ScopeSet::AbsolutePath(ns) => (ns, None, false),
             ScopeSet::Macro(macro_kind) => (MacroNS, Some(macro_kind), false),
+            ScopeSet::Late(ns, ..) => (ns, None, false),
         };
 
         // This is *the* result, resolution from the scope closest to the resolved identifier.
@@ -745,19 +793,34 @@ impl<'a> Resolver<'a> {
                             Err((Determinacy::Determined, _)) => Err(Determinacy::Determined),
                         }
                     }
-                    Scope::Module(module) => {
+                    Scope::Module(module, derive_fallback_lint_id) => {
                         let adjusted_parent_scope = &ParentScope { module, ..*parent_scope };
                         let binding = this.resolve_ident_in_module_unadjusted_ext(
                             ModuleOrUniformRoot::Module(module),
                             ident,
                             ns,
                             adjusted_parent_scope,
-                            true,
+                            !matches!(scope_set, ScopeSet::Late(..)),
                             record_used,
                             path_span,
                         );
                         match binding {
                             Ok(binding) => {
+                                if let Some(lint_id) = derive_fallback_lint_id {
+                                    this.lint_buffer.buffer_lint_with_diagnostic(
+                                        PROC_MACRO_DERIVE_RESOLUTION_FALLBACK,
+                                        lint_id,
+                                        orig_ident.span,
+                                        &format!(
+                                            "cannot find {} `{}` in this scope",
+                                            ns.descr(),
+                                            ident
+                                        ),
+                                        BuiltinLintDiagnostics::ProcMacroDeriveResolutionFallback(
+                                            orig_ident.span,
+                                        ),
+                                    );
+                                }
                                 let misc_flags = if ptr::eq(module, this.graph_root) {
                                     Flags::MISC_SUGGEST_CRATE
                                 } else if module.is_normal() {
@@ -841,7 +904,7 @@ impl<'a> Resolver<'a> {
                     Ok((binding, flags))
                         if sub_namespace_match(binding.macro_kind(), macro_kind) =>
                     {
-                        if !record_used {
+                        if !record_used || matches!(scope_set, ScopeSet::Late(..)) {
                             return Some(Ok(binding));
                         }
 
diff --git a/compiler/rustc_save_analysis/src/dump_visitor.rs b/compiler/rustc_save_analysis/src/dump_visitor.rs
index 625d7c83120..12c77e0c8a6 100644
--- a/compiler/rustc_save_analysis/src/dump_visitor.rs
+++ b/compiler/rustc_save_analysis/src/dump_visitor.rs
@@ -151,7 +151,7 @@ impl<'tcx> DumpVisitor<'tcx> {
             },
             crate_root: crate_root.unwrap_or_else(|| "<no source>".to_owned()),
             external_crates: self.save_ctxt.get_external_crates(),
-            span: self.span_from_span(krate.item.span),
+            span: self.span_from_span(krate.item.inner),
         };
 
         self.dumper.crate_prelude(data);
@@ -301,7 +301,7 @@ impl<'tcx> DumpVisitor<'tcx> {
 
     fn process_struct_field_def(
         &mut self,
-        field: &'tcx hir::StructField<'tcx>,
+        field: &'tcx hir::FieldDef<'tcx>,
         parent_id: hir::HirId,
     ) {
         let field_data = self.save_ctxt.get_field_data(field, parent_id);
@@ -320,15 +320,6 @@ impl<'tcx> DumpVisitor<'tcx> {
         for param in generics.params {
             match param.kind {
                 hir::GenericParamKind::Lifetime { .. } => {}
-                hir::GenericParamKind::Type {
-                    synthetic: Some(hir::SyntheticTyParamKind::ImplTrait),
-                    ..
-                } => {
-                    return self
-                        .nest_typeck_results(self.tcx.hir().local_def_id(param.hir_id), |this| {
-                            this.visit_generics(generics)
-                        });
-                }
                 hir::GenericParamKind::Type { .. } => {
                     let param_ss = param.name.ident().span;
                     let name = escape(self.span.snippet(param_ss));
@@ -496,6 +487,7 @@ impl<'tcx> DumpVisitor<'tcx> {
 
         if !self.span.filter_generated(item.ident.span) {
             let span = self.span_from_span(item.ident.span);
+            let attrs = self.tcx.hir().attrs(item.hir_id());
             self.dumper.dump_def(
                 &access_from!(self.save_ctxt, item, item.hir_id()),
                 Def {
@@ -508,9 +500,9 @@ impl<'tcx> DumpVisitor<'tcx> {
                     parent: None,
                     children: fields,
                     decl_id: None,
-                    docs: self.save_ctxt.docs_for_attrs(&item.attrs),
+                    docs: self.save_ctxt.docs_for_attrs(attrs),
                     sig: sig::item_signature(item, &self.save_ctxt),
-                    attributes: lower_attributes(item.attrs.to_vec(), &self.save_ctxt),
+                    attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
                 },
             );
         }
@@ -554,6 +546,7 @@ impl<'tcx> DumpVisitor<'tcx> {
                         let span = self.span_from_span(name_span);
                         let id = id_from_hir_id(variant.id, &self.save_ctxt);
                         let parent = Some(id_from_def_id(item.def_id.to_def_id()));
+                        let attrs = self.tcx.hir().attrs(variant.id);
 
                         self.dumper.dump_def(
                             &access,
@@ -567,12 +560,9 @@ impl<'tcx> DumpVisitor<'tcx> {
                                 parent,
                                 children: vec![],
                                 decl_id: None,
-                                docs: self.save_ctxt.docs_for_attrs(&variant.attrs),
+                                docs: self.save_ctxt.docs_for_attrs(attrs),
                                 sig: sig::variant_signature(variant, &self.save_ctxt),
-                                attributes: lower_attributes(
-                                    variant.attrs.to_vec(),
-                                    &self.save_ctxt,
-                                ),
+                                attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
                             },
                         );
                     }
@@ -594,6 +584,7 @@ impl<'tcx> DumpVisitor<'tcx> {
                         let span = self.span_from_span(name_span);
                         let id = id_from_hir_id(variant.id, &self.save_ctxt);
                         let parent = Some(id_from_def_id(item.def_id.to_def_id()));
+                        let attrs = self.tcx.hir().attrs(variant.id);
 
                         self.dumper.dump_def(
                             &access,
@@ -607,12 +598,9 @@ impl<'tcx> DumpVisitor<'tcx> {
                                 parent,
                                 children: vec![],
                                 decl_id: None,
-                                docs: self.save_ctxt.docs_for_attrs(&variant.attrs),
+                                docs: self.save_ctxt.docs_for_attrs(attrs),
                                 sig: sig::variant_signature(variant, &self.save_ctxt),
-                                attributes: lower_attributes(
-                                    variant.attrs.to_vec(),
-                                    &self.save_ctxt,
-                                ),
+                                attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
                             },
                         );
                     }
@@ -675,6 +663,7 @@ impl<'tcx> DumpVisitor<'tcx> {
             let span = self.span_from_span(item.ident.span);
             let children =
                 methods.iter().map(|i| id_from_def_id(i.id.def_id.to_def_id())).collect();
+            let attrs = self.tcx.hir().attrs(item.hir_id());
             self.dumper.dump_def(
                 &access_from!(self.save_ctxt, item, item.hir_id()),
                 Def {
@@ -687,9 +676,9 @@ impl<'tcx> DumpVisitor<'tcx> {
                     parent: None,
                     children,
                     decl_id: None,
-                    docs: self.save_ctxt.docs_for_attrs(&item.attrs),
+                    docs: self.save_ctxt.docs_for_attrs(attrs),
                     sig: sig::item_signature(item, &self.save_ctxt),
-                    attributes: lower_attributes(item.attrs.to_vec(), &self.save_ctxt),
+                    attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
                 },
             );
         }
@@ -795,7 +784,7 @@ impl<'tcx> DumpVisitor<'tcx> {
         &mut self,
         ex: &'tcx hir::Expr<'tcx>,
         path: &'tcx hir::QPath<'tcx>,
-        fields: &'tcx [hir::Field<'tcx>],
+        fields: &'tcx [hir::ExprField<'tcx>],
         variant: &'tcx ty::VariantDef,
         rest: Option<&'tcx hir::Expr<'tcx>>,
     ) {
@@ -998,6 +987,7 @@ impl<'tcx> DumpVisitor<'tcx> {
             hir::TraitItemKind::Const(ref ty, body) => {
                 let body = body.map(|b| &self.tcx.hir().body(b).value);
                 let respan = respan(vis_span, hir::VisibilityKind::Public);
+                let attrs = self.tcx.hir().attrs(trait_item.hir_id());
                 self.process_assoc_const(
                     trait_item.hir_id(),
                     trait_item.ident,
@@ -1005,7 +995,7 @@ impl<'tcx> DumpVisitor<'tcx> {
                     body,
                     trait_id,
                     &respan,
-                    &trait_item.attrs,
+                    attrs,
                 );
             }
             hir::TraitItemKind::Fn(ref sig, ref trait_fn) => {
@@ -1031,6 +1021,7 @@ impl<'tcx> DumpVisitor<'tcx> {
                 if !self.span.filter_generated(trait_item.ident.span) {
                     let span = self.span_from_span(trait_item.ident.span);
                     let id = id_from_def_id(trait_item.def_id.to_def_id());
+                    let attrs = self.tcx.hir().attrs(trait_item.hir_id());
 
                     self.dumper.dump_def(
                         &Access { public: true, reachable: true },
@@ -1044,7 +1035,7 @@ impl<'tcx> DumpVisitor<'tcx> {
                             parent: Some(id_from_def_id(trait_id)),
                             children: vec![],
                             decl_id: None,
-                            docs: self.save_ctxt.docs_for_attrs(&trait_item.attrs),
+                            docs: self.save_ctxt.docs_for_attrs(attrs),
                             sig: sig::assoc_type_signature(
                                 trait_item.hir_id(),
                                 trait_item.ident,
@@ -1052,10 +1043,7 @@ impl<'tcx> DumpVisitor<'tcx> {
                                 default_ty.as_ref().map(|ty| &**ty),
                                 &self.save_ctxt,
                             ),
-                            attributes: lower_attributes(
-                                trait_item.attrs.to_vec(),
-                                &self.save_ctxt,
-                            ),
+                            attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
                         },
                     );
                 }
@@ -1072,6 +1060,7 @@ impl<'tcx> DumpVisitor<'tcx> {
         match impl_item.kind {
             hir::ImplItemKind::Const(ref ty, body) => {
                 let body = self.tcx.hir().body(body);
+                let attrs = self.tcx.hir().attrs(impl_item.hir_id());
                 self.process_assoc_const(
                     impl_item.hir_id(),
                     impl_item.ident,
@@ -1079,7 +1068,7 @@ impl<'tcx> DumpVisitor<'tcx> {
                     Some(&body.value),
                     impl_id,
                     &impl_item.vis,
-                    &impl_item.attrs,
+                    attrs,
                 );
             }
             hir::ImplItemKind::Fn(ref sig, body) => {
@@ -1108,16 +1097,12 @@ impl<'tcx> DumpVisitor<'tcx> {
             format!("::{}", self.tcx.def_path_str(self.tcx.hir().local_def_id(id).to_def_id()));
 
         let sm = self.tcx.sess.source_map();
-        let filename = sm.span_to_filename(krate.item.span);
+        let filename = sm.span_to_filename(krate.item.inner);
         let data_id = id_from_hir_id(id, &self.save_ctxt);
-        let children = krate
-            .item
-            .module
-            .item_ids
-            .iter()
-            .map(|i| id_from_def_id(i.def_id.to_def_id()))
-            .collect();
-        let span = self.span_from_span(krate.item.span);
+        let children =
+            krate.item.item_ids.iter().map(|i| id_from_def_id(i.def_id.to_def_id())).collect();
+        let span = self.span_from_span(krate.item.inner);
+        let attrs = self.tcx.hir().attrs(id);
 
         self.dumper.dump_def(
             &Access { public: true, reachable: true },
@@ -1131,9 +1116,9 @@ impl<'tcx> DumpVisitor<'tcx> {
                 children,
                 parent: None,
                 decl_id: None,
-                docs: self.save_ctxt.docs_for_attrs(krate.item.attrs),
+                docs: self.save_ctxt.docs_for_attrs(attrs),
                 sig: None,
-                attributes: lower_attributes(krate.item.attrs.to_owned(), &self.save_ctxt),
+                attributes: lower_attributes(attrs.to_owned(), &self.save_ctxt),
             },
         );
         intravisit::walk_crate(self, krate);
@@ -1263,6 +1248,7 @@ impl<'tcx> Visitor<'tcx> for DumpVisitor<'tcx> {
                 if !self.span.filter_generated(item.ident.span) {
                     let span = self.span_from_span(item.ident.span);
                     let id = id_from_def_id(item.def_id.to_def_id());
+                    let attrs = self.tcx.hir().attrs(item.hir_id());
 
                     self.dumper.dump_def(
                         &access_from!(self.save_ctxt, item, item.hir_id()),
@@ -1276,9 +1262,9 @@ impl<'tcx> Visitor<'tcx> for DumpVisitor<'tcx> {
                             parent: None,
                             children: vec![],
                             decl_id: None,
-                            docs: self.save_ctxt.docs_for_attrs(&item.attrs),
+                            docs: self.save_ctxt.docs_for_attrs(attrs),
                             sig: sig::item_signature(item, &self.save_ctxt),
-                            attributes: lower_attributes(item.attrs.to_vec(), &self.save_ctxt),
+                            attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
                         },
                     );
                 }
diff --git a/compiler/rustc_save_analysis/src/lib.rs b/compiler/rustc_save_analysis/src/lib.rs
index b3f281bcabc..c19c16b88a7 100644
--- a/compiler/rustc_save_analysis/src/lib.rs
+++ b/compiler/rustc_save_analysis/src/lib.rs
@@ -1,6 +1,6 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 mod dump_visitor;
@@ -139,6 +139,7 @@ impl<'tcx> SaveContext<'tcx> {
     pub fn get_extern_item_data(&self, item: &hir::ForeignItem<'_>) -> Option<Data> {
         let def_id = item.def_id.to_def_id();
         let qualname = format!("::{}", self.tcx.def_path_str(def_id));
+        let attrs = self.tcx.hir().attrs(item.hir_id());
         match item.kind {
             hir::ForeignItemKind::Fn(ref decl, arg_names, ref generics) => {
                 filter!(self.span_utils, item.ident.span);
@@ -169,9 +170,9 @@ impl<'tcx> SaveContext<'tcx> {
                     parent: None,
                     children: vec![],
                     decl_id: None,
-                    docs: self.docs_for_attrs(&item.attrs),
+                    docs: self.docs_for_attrs(attrs),
                     sig: sig::foreign_item_signature(item, self),
-                    attributes: lower_attributes(item.attrs.to_vec(), self),
+                    attributes: lower_attributes(attrs.to_vec(), self),
                 }))
             }
             hir::ForeignItemKind::Static(ref ty, _) => {
@@ -190,9 +191,9 @@ impl<'tcx> SaveContext<'tcx> {
                     parent: None,
                     children: vec![],
                     decl_id: None,
-                    docs: self.docs_for_attrs(&item.attrs),
+                    docs: self.docs_for_attrs(attrs),
                     sig: sig::foreign_item_signature(item, self),
-                    attributes: lower_attributes(item.attrs.to_vec(), self),
+                    attributes: lower_attributes(attrs.to_vec(), self),
                 }))
             }
             // FIXME(plietar): needs a new DefKind in rls-data
@@ -202,6 +203,7 @@ impl<'tcx> SaveContext<'tcx> {
 
     pub fn get_item_data(&self, item: &hir::Item<'_>) -> Option<Data> {
         let def_id = item.def_id.to_def_id();
+        let attrs = self.tcx.hir().attrs(item.hir_id());
         match item.kind {
             hir::ItemKind::Fn(ref sig, ref generics, _) => {
                 let qualname = format!("::{}", self.tcx.def_path_str(def_id));
@@ -224,9 +226,9 @@ impl<'tcx> SaveContext<'tcx> {
                     parent: None,
                     children: vec![],
                     decl_id: None,
-                    docs: self.docs_for_attrs(&item.attrs),
+                    docs: self.docs_for_attrs(attrs),
                     sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(item.attrs.to_vec(), self),
+                    attributes: lower_attributes(attrs.to_vec(), self),
                 }))
             }
             hir::ItemKind::Static(ref typ, ..) => {
@@ -247,9 +249,9 @@ impl<'tcx> SaveContext<'tcx> {
                     parent: None,
                     children: vec![],
                     decl_id: None,
-                    docs: self.docs_for_attrs(&item.attrs),
+                    docs: self.docs_for_attrs(attrs),
                     sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(item.attrs.to_vec(), self),
+                    attributes: lower_attributes(attrs.to_vec(), self),
                 }))
             }
             hir::ItemKind::Const(ref typ, _) => {
@@ -269,9 +271,9 @@ impl<'tcx> SaveContext<'tcx> {
                     parent: None,
                     children: vec![],
                     decl_id: None,
-                    docs: self.docs_for_attrs(&item.attrs),
+                    docs: self.docs_for_attrs(attrs),
                     sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(item.attrs.to_vec(), self),
+                    attributes: lower_attributes(attrs.to_vec(), self),
                 }))
             }
             hir::ItemKind::Mod(ref m) => {
@@ -296,9 +298,9 @@ impl<'tcx> SaveContext<'tcx> {
                         .map(|i| id_from_def_id(i.def_id.to_def_id()))
                         .collect(),
                     decl_id: None,
-                    docs: self.docs_for_attrs(&item.attrs),
+                    docs: self.docs_for_attrs(attrs),
                     sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(item.attrs.to_vec(), self),
+                    attributes: lower_attributes(attrs.to_vec(), self),
                 }))
             }
             hir::ItemKind::Enum(ref def, ref generics) => {
@@ -317,9 +319,9 @@ impl<'tcx> SaveContext<'tcx> {
                     parent: None,
                     children: def.variants.iter().map(|v| id_from_hir_id(v.id, self)).collect(),
                     decl_id: None,
-                    docs: self.docs_for_attrs(&item.attrs),
+                    docs: self.docs_for_attrs(attrs),
                     sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(item.attrs.to_vec(), self),
+                    attributes: lower_attributes(attrs.to_vec(), self),
                 }))
             }
             hir::ItemKind::Impl(hir::Impl { ref of_trait, ref self_ty, ref items, .. }) => {
@@ -377,7 +379,7 @@ impl<'tcx> SaveContext<'tcx> {
         }
     }
 
-    pub fn get_field_data(&self, field: &hir::StructField<'_>, scope: hir::HirId) -> Option<Def> {
+    pub fn get_field_data(&self, field: &hir::FieldDef<'_>, scope: hir::HirId) -> Option<Def> {
         let name = field.ident.to_string();
         let scope_def_id = self.tcx.hir().local_def_id(scope).to_def_id();
         let qualname = format!("::{}::{}", self.tcx.def_path_str(scope_def_id), field.ident);
@@ -387,6 +389,7 @@ impl<'tcx> SaveContext<'tcx> {
 
         let id = id_from_def_id(field_def_id);
         let span = self.span_from_span(field.ident.span);
+        let attrs = self.tcx.hir().attrs(field.hir_id);
 
         Some(Def {
             kind: DefKind::Field,
@@ -398,9 +401,9 @@ impl<'tcx> SaveContext<'tcx> {
             parent: Some(id_from_def_id(scope_def_id)),
             children: vec![],
             decl_id: None,
-            docs: self.docs_for_attrs(&field.attrs),
+            docs: self.docs_for_attrs(attrs),
             sig: sig::field_signature(field, self),
-            attributes: lower_attributes(field.attrs.to_vec(), self),
+            attributes: lower_attributes(attrs.to_vec(), self),
         })
     }
 
@@ -424,9 +427,9 @@ impl<'tcx> SaveContext<'tcx> {
                             let trait_id = self.tcx.trait_id_of_impl(impl_id);
                             let mut docs = String::new();
                             let mut attrs = vec![];
-                            if let Some(Node::ImplItem(item)) = hir.find(hir_id) {
-                                docs = self.docs_for_attrs(&item.attrs);
-                                attrs = item.attrs.to_vec();
+                            if let Some(Node::ImplItem(_)) = hir.find(hir_id) {
+                                attrs = self.tcx.hir().attrs(hir_id).to_vec();
+                                docs = self.docs_for_attrs(&attrs);
                             }
 
                             let mut decl_id = None;
@@ -470,9 +473,9 @@ impl<'tcx> SaveContext<'tcx> {
                         let mut docs = String::new();
                         let mut attrs = vec![];
 
-                        if let Some(Node::TraitItem(item)) = self.tcx.hir().find(hir_id) {
-                            docs = self.docs_for_attrs(&item.attrs);
-                            attrs = item.attrs.to_vec();
+                        if let Some(Node::TraitItem(_)) = self.tcx.hir().find(hir_id) {
+                            attrs = self.tcx.hir().attrs(hir_id).to_vec();
+                            docs = self.docs_for_attrs(&attrs);
                         }
 
                         (
@@ -513,19 +516,6 @@ impl<'tcx> SaveContext<'tcx> {
         })
     }
 
-    pub fn get_trait_ref_data(&self, trait_ref: &hir::TraitRef<'_>) -> Option<Ref> {
-        self.lookup_def_id(trait_ref.hir_ref_id).and_then(|def_id| {
-            let span = trait_ref.path.span;
-            if generated_code(span) {
-                return None;
-            }
-            let sub_span = trait_ref.path.segments.last().unwrap().ident.span;
-            filter!(self.span_utils, sub_span);
-            let span = self.span_from_span(sub_span);
-            Some(Ref { kind: RefKind::Type, span, ref_id: id_from_def_id(def_id) })
-        })
-    }
-
     pub fn get_expr_data(&self, expr: &hir::Expr<'_>) -> Option<Data> {
         let ty = self.typeck_results().expr_ty_adjusted_opt(expr)?;
         if matches!(ty.kind(), ty::Error(_)) {
@@ -766,7 +756,7 @@ impl<'tcx> SaveContext<'tcx> {
 
     pub fn get_field_ref_data(
         &self,
-        field_ref: &hir::Field<'_>,
+        field_ref: &hir::ExprField<'_>,
         variant: &ty::VariantDef,
     ) -> Option<Ref> {
         filter!(self.span_utils, field_ref.ident.span);
@@ -781,7 +771,10 @@ impl<'tcx> SaveContext<'tcx> {
     /// For a given piece of AST defined by the supplied Span and NodeId,
     /// returns `None` if the node is not macro-generated or the span is malformed,
     /// else uses the expansion callsite and callee to return some MacroRef.
-    pub fn get_macro_use_data(&self, span: Span) -> Option<MacroRef> {
+    ///
+    /// FIXME: [`DumpVisitor::process_macro_use`] should actually dump this data
+    #[allow(dead_code)]
+    fn get_macro_use_data(&self, span: Span) -> Option<MacroRef> {
         if !generated_code(span) {
             return None;
         }
diff --git a/compiler/rustc_save_analysis/src/sig.rs b/compiler/rustc_save_analysis/src/sig.rs
index 33db189af37..c3bc1c191ff 100644
--- a/compiler/rustc_save_analysis/src/sig.rs
+++ b/compiler/rustc_save_analysis/src/sig.rs
@@ -55,7 +55,7 @@ pub fn foreign_item_signature(
 
 /// Signature for a struct or tuple field declaration.
 /// Does not include a trailing comma.
-pub fn field_signature(field: &hir::StructField<'_>, scx: &SaveContext<'_>) -> Option<Signature> {
+pub fn field_signature(field: &hir::FieldDef<'_>, scx: &SaveContext<'_>) -> Option<Signature> {
     if !scx.config.signatures {
         return None;
     }
@@ -614,11 +614,12 @@ impl<'hir> Sig for hir::Generics<'hir> {
                 start: offset + text.len(),
                 end: offset + text.len() + param_text.as_str().len(),
             });
-            if let hir::GenericParamKind::Const { ref ty, ref default } = param.kind {
+            if let hir::GenericParamKind::Const { ref ty, default } = param.kind {
                 param_text.push_str(": ");
                 param_text.push_str(&ty_to_string(&ty));
-                if let Some(ref _default) = default {
-                    // FIXME(const_generics_defaults): push the `default` value here
+                if let Some(default) = default {
+                    param_text.push_str(" = ");
+                    param_text.push_str(&id_to_string(&scx.tcx.hir(), default.hir_id));
                 }
             }
             if !param.bounds.is_empty() {
@@ -655,7 +656,7 @@ impl<'hir> Sig for hir::Generics<'hir> {
     }
 }
 
-impl<'hir> Sig for hir::StructField<'hir> {
+impl<'hir> Sig for hir::FieldDef<'hir> {
     fn make(&self, offset: usize, _parent_id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
         let mut text = String::new();
 
diff --git a/compiler/rustc_serialize/src/json.rs b/compiler/rustc_serialize/src/json.rs
index bbbe568f17a..78a102c5c23 100644
--- a/compiler/rustc_serialize/src/json.rs
+++ b/compiler/rustc_serialize/src/json.rs
@@ -553,6 +553,12 @@ impl<'a> crate::Encoder for Encoder<'a> {
     fn emit_str(&mut self, v: &str) -> EncodeResult {
         escape_str(self.writer, v)
     }
+    fn emit_raw_bytes(&mut self, s: &[u8]) -> Result<(), Self::Error> {
+        for &c in s.iter() {
+            self.emit_u8(c)?;
+        }
+        Ok(())
+    }
 
     fn emit_enum<F>(&mut self, _name: &str, f: F) -> EncodeResult
     where
@@ -879,6 +885,12 @@ impl<'a> crate::Encoder for PrettyEncoder<'a> {
     fn emit_str(&mut self, v: &str) -> EncodeResult {
         escape_str(self.writer, v)
     }
+    fn emit_raw_bytes(&mut self, s: &[u8]) -> Result<(), Self::Error> {
+        for &c in s.iter() {
+            self.emit_u8(c)?;
+        }
+        Ok(())
+    }
 
     fn emit_enum<F>(&mut self, _name: &str, f: F) -> EncodeResult
     where
@@ -2354,6 +2366,13 @@ impl crate::Decoder for Decoder {
         expect!(self.pop(), String).map(Cow::Owned)
     }
 
+    fn read_raw_bytes_into(&mut self, s: &mut [u8]) -> Result<(), Self::Error> {
+        for c in s.iter_mut() {
+            *c = self.read_u8()?;
+        }
+        Ok(())
+    }
+
     fn read_enum<T, F>(&mut self, _name: &str, f: F) -> DecodeResult<T>
     where
         F: FnOnce(&mut Decoder) -> DecodeResult<T>,
diff --git a/compiler/rustc_serialize/src/lib.rs b/compiler/rustc_serialize/src/lib.rs
index e439ddcdaa9..cf5a9118275 100644
--- a/compiler/rustc_serialize/src/lib.rs
+++ b/compiler/rustc_serialize/src/lib.rs
@@ -16,7 +16,8 @@ Core encoding and decoding interfaces.
 #![feature(min_specialization)]
 #![feature(vec_spare_capacity)]
 #![feature(core_intrinsics)]
-#![feature(int_bits_const)]
+#![feature(maybe_uninit_array_assume_init)]
+#![feature(maybe_uninit_uninit_array)]
 #![feature(maybe_uninit_slice)]
 #![feature(new_uninit)]
 #![cfg_attr(test, feature(test))]
diff --git a/compiler/rustc_serialize/src/opaque.rs b/compiler/rustc_serialize/src/opaque.rs
index 3e37fc87ce6..6e36184aff0 100644
--- a/compiler/rustc_serialize/src/opaque.rs
+++ b/compiler/rustc_serialize/src/opaque.rs
@@ -1,6 +1,7 @@
 use crate::leb128::{self, max_leb128_len};
-use crate::serialize;
+use crate::serialize::{self, Encoder as _};
 use std::borrow::Cow;
+use std::convert::TryInto;
 use std::fs::File;
 use std::io::{self, Write};
 use std::mem::MaybeUninit;
@@ -30,11 +31,6 @@ impl Encoder {
     pub fn position(&self) -> usize {
         self.data.len()
     }
-
-    #[inline]
-    pub fn emit_raw_bytes(&mut self, s: &[u8]) {
-        self.data.extend_from_slice(s);
-    }
 }
 
 macro_rules! write_leb128 {
@@ -154,7 +150,12 @@ impl serialize::Encoder for Encoder {
     #[inline]
     fn emit_str(&mut self, v: &str) -> EncodeResult {
         self.emit_usize(v.len())?;
-        self.emit_raw_bytes(v.as_bytes());
+        self.emit_raw_bytes(v.as_bytes())
+    }
+
+    #[inline]
+    fn emit_raw_bytes(&mut self, s: &[u8]) -> EncodeResult {
+        self.data.extend_from_slice(s);
         Ok(())
     }
 }
@@ -208,11 +209,6 @@ impl FileEncoder {
         self.flushed + self.buffered
     }
 
-    #[inline]
-    pub fn emit_raw_bytes(&mut self, s: &[u8]) -> FileEncodeResult {
-        self.write_all(s)
-    }
-
     pub fn flush(&mut self) -> FileEncodeResult {
         // This is basically a copy of `BufWriter::flush`. If `BufWriter` ever
         // offers a raw buffer access API, we can use it, and remove this.
@@ -508,6 +504,11 @@ impl serialize::Encoder for FileEncoder {
         self.emit_usize(v.len())?;
         self.emit_raw_bytes(v.as_bytes())
     }
+
+    #[inline]
+    fn emit_raw_bytes(&mut self, s: &[u8]) -> FileEncodeResult {
+        self.write_all(s)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -541,23 +542,10 @@ impl<'a> Decoder<'a> {
     }
 
     #[inline]
-    pub fn read_raw_bytes(&mut self, s: &mut [MaybeUninit<u8>]) -> Result<(), String> {
+    pub fn read_raw_bytes(&mut self, bytes: usize) -> &'a [u8] {
         let start = self.position;
-        let end = start + s.len();
-        assert!(end <= self.data.len());
-
-        // SAFETY: Both `src` and `dst` point to at least `s.len()` elements:
-        // `src` points to at least `s.len()` elements by above assert, and
-        // `dst` points to `s.len()` elements by derivation from `s`.
-        unsafe {
-            let src = self.data.as_ptr().add(start);
-            let dst = s.as_mut_ptr() as *mut u8;
-            ptr::copy_nonoverlapping(src, dst, s.len());
-        }
-
-        self.position = end;
-
-        Ok(())
+        self.position += bytes;
+        &self.data[start..self.position]
     }
 }
 
@@ -677,6 +665,14 @@ impl<'a> serialize::Decoder for Decoder<'a> {
     fn error(&mut self, err: &str) -> Self::Error {
         err.to_string()
     }
+
+    #[inline]
+    fn read_raw_bytes_into(&mut self, s: &mut [u8]) -> Result<(), String> {
+        let start = self.position;
+        self.position += s.len();
+        s.copy_from_slice(&self.data[start..self.position]);
+        Ok(())
+    }
 }
 
 // Specializations for contiguous byte sequences follow. The default implementations for slices
@@ -689,8 +685,7 @@ impl<'a> serialize::Decoder for Decoder<'a> {
 impl serialize::Encodable<Encoder> for [u8] {
     fn encode(&self, e: &mut Encoder) -> EncodeResult {
         serialize::Encoder::emit_usize(e, self.len())?;
-        e.emit_raw_bytes(self);
-        Ok(())
+        e.emit_raw_bytes(self)
     }
 }
 
@@ -706,15 +701,48 @@ impl serialize::Encodable<FileEncoder> for [u8] {
 impl<'a> serialize::Decodable<Decoder<'a>> for Vec<u8> {
     fn decode(d: &mut Decoder<'a>) -> Result<Self, String> {
         let len = serialize::Decoder::read_usize(d)?;
+        Ok(d.read_raw_bytes(len).to_owned())
+    }
+}
 
-        let mut v = Vec::with_capacity(len);
-        let buf = &mut v.spare_capacity_mut()[..len];
-        d.read_raw_bytes(buf)?;
+// An integer that will always encode to 8 bytes.
+pub struct IntEncodedWithFixedSize(pub u64);
 
-        unsafe {
-            v.set_len(len);
-        }
+impl IntEncodedWithFixedSize {
+    pub const ENCODED_SIZE: usize = 8;
+}
+
+impl serialize::Encodable<Encoder> for IntEncodedWithFixedSize {
+    #[inline]
+    fn encode(&self, e: &mut Encoder) -> EncodeResult {
+        let _start_pos = e.position();
+        e.emit_raw_bytes(&self.0.to_le_bytes())?;
+        let _end_pos = e.position();
+        debug_assert_eq!((_end_pos - _start_pos), IntEncodedWithFixedSize::ENCODED_SIZE);
+        Ok(())
+    }
+}
+
+impl serialize::Encodable<FileEncoder> for IntEncodedWithFixedSize {
+    #[inline]
+    fn encode(&self, e: &mut FileEncoder) -> FileEncodeResult {
+        let _start_pos = e.position();
+        e.emit_raw_bytes(&self.0.to_le_bytes())?;
+        let _end_pos = e.position();
+        debug_assert_eq!((_end_pos - _start_pos), IntEncodedWithFixedSize::ENCODED_SIZE);
+        Ok(())
+    }
+}
+
+impl<'a> serialize::Decodable<Decoder<'a>> for IntEncodedWithFixedSize {
+    #[inline]
+    fn decode(decoder: &mut Decoder<'a>) -> Result<IntEncodedWithFixedSize, String> {
+        let _start_pos = decoder.position();
+        let bytes = decoder.read_raw_bytes(IntEncodedWithFixedSize::ENCODED_SIZE);
+        let _end_pos = decoder.position();
+        debug_assert_eq!((_end_pos - _start_pos), IntEncodedWithFixedSize::ENCODED_SIZE);
 
-        Ok(v)
+        let value = u64::from_le_bytes(bytes.try_into().unwrap());
+        Ok(IntEncodedWithFixedSize(value))
     }
 }
diff --git a/compiler/rustc_serialize/src/serialize.rs b/compiler/rustc_serialize/src/serialize.rs
index 47aad5b88c6..d3e5f306970 100644
--- a/compiler/rustc_serialize/src/serialize.rs
+++ b/compiler/rustc_serialize/src/serialize.rs
@@ -33,6 +33,7 @@ pub trait Encoder {
     fn emit_f32(&mut self, v: f32) -> Result<(), Self::Error>;
     fn emit_char(&mut self, v: char) -> Result<(), Self::Error>;
     fn emit_str(&mut self, v: &str) -> Result<(), Self::Error>;
+    fn emit_raw_bytes(&mut self, s: &[u8]) -> Result<(), Self::Error>;
 
     // Compound types:
     #[inline]
@@ -224,6 +225,7 @@ pub trait Decoder {
     fn read_f32(&mut self) -> Result<f32, Self::Error>;
     fn read_char(&mut self) -> Result<char, Self::Error>;
     fn read_str(&mut self) -> Result<Cow<'_, str>, Self::Error>;
+    fn read_raw_bytes_into(&mut self, s: &mut [u8]) -> Result<(), Self::Error>;
 
     // Compound types:
     #[inline]
diff --git a/compiler/rustc_serialize/tests/leb128.rs b/compiler/rustc_serialize/tests/leb128.rs
index a2bcf2c251d..3e2aab5125a 100644
--- a/compiler/rustc_serialize/tests/leb128.rs
+++ b/compiler/rustc_serialize/tests/leb128.rs
@@ -1,4 +1,3 @@
-#![feature(int_bits_const)]
 #![feature(maybe_uninit_slice)]
 #![feature(maybe_uninit_uninit_array)]
 
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 65be14e3510..1c9bd153f8d 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -10,7 +10,6 @@ use crate::{early_error, early_warn, Session};
 
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::impl_stable_hash_via_hash;
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 
 use rustc_target::abi::{Align, TargetDataLayout};
 use rustc_target::spec::{SplitDebuginfo, Target, TargetTriple};
@@ -19,7 +18,7 @@ use rustc_serialize::json;
 
 use crate::parse::CrateConfig;
 use rustc_feature::UnstableFeatures;
-use rustc_span::edition::{Edition, DEFAULT_EDITION, EDITION_NAME_LIST};
+use rustc_span::edition::{Edition, DEFAULT_EDITION, EDITION_NAME_LIST, LATEST_STABLE_EDITION};
 use rustc_span::source_map::{FileName, FilePathMapping};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::SourceFileHashAlgorithm;
@@ -36,66 +35,6 @@ use std::iter::{self, FromIterator};
 use std::path::{Path, PathBuf};
 use std::str::{self, FromStr};
 
-bitflags! {
-    #[derive(Default, Encodable, Decodable)]
-    pub struct SanitizerSet: u8 {
-        const ADDRESS = 1 << 0;
-        const LEAK    = 1 << 1;
-        const MEMORY  = 1 << 2;
-        const THREAD  = 1 << 3;
-        const HWADDRESS  = 1 << 4;
-    }
-}
-
-/// Formats a sanitizer set as a comma separated list of sanitizers' names.
-impl fmt::Display for SanitizerSet {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut first = true;
-        for s in *self {
-            let name = match s {
-                SanitizerSet::ADDRESS => "address",
-                SanitizerSet::LEAK => "leak",
-                SanitizerSet::MEMORY => "memory",
-                SanitizerSet::THREAD => "thread",
-                SanitizerSet::HWADDRESS => "hwaddress",
-                _ => panic!("unrecognized sanitizer {:?}", s),
-            };
-            if !first {
-                f.write_str(",")?;
-            }
-            f.write_str(name)?;
-            first = false;
-        }
-        Ok(())
-    }
-}
-
-impl IntoIterator for SanitizerSet {
-    type Item = SanitizerSet;
-    type IntoIter = std::vec::IntoIter<SanitizerSet>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        [
-            SanitizerSet::ADDRESS,
-            SanitizerSet::LEAK,
-            SanitizerSet::MEMORY,
-            SanitizerSet::THREAD,
-            SanitizerSet::HWADDRESS,
-        ]
-        .iter()
-        .copied()
-        .filter(|&s| self.contains(s))
-        .collect::<Vec<_>>()
-        .into_iter()
-    }
-}
-
-impl<CTX> HashStable<CTX> for SanitizerSet {
-    fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
-        self.bits().hash_stable(ctx, hasher);
-    }
-}
-
 /// The different settings that the `-Z strip` flag can have.
 #[derive(Clone, Copy, PartialEq, Hash, Debug)]
 pub enum Strip {
@@ -136,19 +75,21 @@ impl_stable_hash_via_hash!(OptLevel);
 
 /// This is what the `LtoCli` values get mapped to after resolving defaults and
 /// and taking other command line options into account.
+///
+/// Note that linker plugin-based LTO is a different mechanism entirely.
 #[derive(Clone, PartialEq)]
 pub enum Lto {
-    /// Don't do any LTO whatsoever
+    /// Don't do any LTO whatsoever.
     No,
 
-    /// Do a full crate graph LTO with ThinLTO
+    /// Do a full-crate-graph (inter-crate) LTO with ThinLTO.
     Thin,
 
-    /// Do a local graph LTO with ThinLTO (only relevant for multiple codegen
-    /// units).
+    /// Do a local ThinLTO (intra-crate, over the CodeGen Units of the local crate only). This is
+    /// only relevant if multiple CGUs are used.
     ThinLocal,
 
-    /// Do a full crate graph LTO with "fat" LTO
+    /// Do a full-crate-graph (inter-crate) LTO with "fat" LTO.
     Fat,
 }
 
@@ -184,6 +125,37 @@ pub enum MirSpanview {
     Block,
 }
 
+/// The different settings that the `-Z instrument-coverage` flag can have.
+///
+/// Coverage instrumentation now supports combining `-Z instrument-coverage`
+/// with compiler and linker optimization (enabled with `-O` or `-C opt-level=1`
+/// and higher). Nevertheless, there are many variables, depending on options
+/// selected, code structure, and enabled attributes. If errors are encountered,
+/// either while compiling or when generating `llvm-cov show` reports, consider
+/// lowering the optimization level, including or excluding `-C link-dead-code`,
+/// or using `-Z instrument-coverage=except-unused-functions` or `-Z
+/// instrument-coverage=except-unused-generics`.
+///
+/// Note that `ExceptUnusedFunctions` means: When `mapgen.rs` generates the
+/// coverage map, it will not attempt to generate synthetic functions for unused
+/// (and not code-generated) functions (whether they are generic or not). As a
+/// result, non-codegenned functions will not be included in the coverage map,
+/// and will not appear, as covered or uncovered, in coverage reports.
+///
+/// `ExceptUnusedGenerics` will add synthetic functions to the coverage map,
+/// unless the function has type parameters.
+#[derive(Clone, Copy, PartialEq, Hash, Debug)]
+pub enum InstrumentCoverage {
+    /// Default `-Z instrument-coverage` or `-Z instrument-coverage=statement`
+    All,
+    /// `-Z instrument-coverage=except-unused-generics`
+    ExceptUnusedGenerics,
+    /// `-Z instrument-coverage=except-unused-functions`
+    ExceptUnusedFunctions,
+    /// `-Z instrument-coverage=off` (or `no`, etc.)
+    Off,
+}
+
 #[derive(Clone, PartialEq, Hash)]
 pub enum LinkerPluginLto {
     LinkerPlugin(PathBuf),
@@ -474,6 +446,7 @@ impl<'a> From<&'a ExternDepSpec> for rustc_lint_defs::ExternDepSpec {
 }
 
 impl Externs {
+    /// Used for testing.
     pub fn new(data: BTreeMap<String, ExternEntry>) -> Externs {
         Externs(data)
     }
@@ -485,6 +458,10 @@ impl Externs {
     pub fn iter(&self) -> BTreeMapIter<'_, String, ExternEntry> {
         self.0.iter()
     }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
 }
 
 impl ExternEntry {
@@ -573,13 +550,6 @@ impl Input {
         }
     }
 
-    pub fn get_input(&mut self) -> Option<&mut String> {
-        match *self {
-            Input::File(_) => None,
-            Input::Str { ref mut input, .. } => Some(input),
-        }
-    }
-
     pub fn source_name(&self) -> FileName {
         match *self {
             Input::File(ref ifile) => ifile.clone().into(),
@@ -734,6 +704,7 @@ impl Default for Options {
             remap_path_prefix: Vec::new(),
             edition: DEFAULT_EDITION,
             json_artifact_notifications: false,
+            json_unused_externs: false,
             pretty: None,
         }
     }
@@ -747,12 +718,6 @@ impl Options {
             || self.debugging_opts.query_dep_graph
     }
 
-    #[inline(always)]
-    pub fn enable_dep_node_debug_strs(&self) -> bool {
-        cfg!(debug_assertions)
-            && (self.debugging_opts.query_dep_graph || self.debugging_opts.incremental_info)
-    }
-
     pub fn file_path_mapping(&self) -> FilePathMapping {
         FilePathMapping::new(self.remap_path_prefix.clone())
     }
@@ -832,7 +797,7 @@ pub const fn default_lib_output() -> CrateType {
     CrateType::Rlib
 }
 
-pub fn default_configuration(sess: &Session) -> CrateConfig {
+fn default_configuration(sess: &Session) -> CrateConfig {
     let end = &sess.target.endian;
     let arch = &sess.target.arch;
     let wordsz = sess.target.pointer_width.to_string();
@@ -927,7 +892,7 @@ pub fn build_configuration(sess: &Session, mut user_cfg: CrateConfig) -> CrateCo
     user_cfg
 }
 
-pub fn build_target_config(
+pub(super) fn build_target_config(
     opts: &Options,
     target_override: Option<Target>,
     sysroot: &PathBuf,
@@ -939,7 +904,7 @@ pub fn build_target_config(
             opts.error_format,
             &format!(
                 "Error loading target specification: {}. \
-            Use `--print target-list` for a list of built-in targets",
+                 Run `rustc --print target-list` for a list of built-in targets",
                 e
             ),
         )
@@ -1034,9 +999,6 @@ mod opt {
     pub fn flag_s(a: S, b: S, c: S) -> R {
         stable(longer(a, b), move |opts| opts.optflag(a, b, c))
     }
-    pub fn flagopt_s(a: S, b: S, c: S, d: S) -> R {
-        stable(longer(a, b), move |opts| opts.optflagopt(a, b, c, d))
-    }
     pub fn flagmulti_s(a: S, b: S, c: S) -> R {
         stable(longer(a, b), move |opts| opts.optflagmulti(a, b, c))
     }
@@ -1047,15 +1009,6 @@ mod opt {
     pub fn multi(a: S, b: S, c: S, d: S) -> R {
         unstable(longer(a, b), move |opts| opts.optmulti(a, b, c, d))
     }
-    pub fn flag(a: S, b: S, c: S) -> R {
-        unstable(longer(a, b), move |opts| opts.optflag(a, b, c))
-    }
-    pub fn flagopt(a: S, b: S, c: S, d: S) -> R {
-        unstable(longer(a, b), move |opts| opts.optflagopt(a, b, c, d))
-    }
-    pub fn flagmulti(a: S, b: S, c: S) -> R {
-        unstable(longer(a, b), move |opts| opts.optflagmulti(a, b, c))
-    }
 }
 
 /// Returns the "short" subset of the rustc command line options,
@@ -1255,15 +1208,23 @@ pub fn parse_color(matches: &getopts::Matches) -> ColorConfig {
     }
 }
 
+/// Possible json config files
+pub struct JsonConfig {
+    pub json_rendered: HumanReadableErrorType,
+    pub json_artifact_notifications: bool,
+    pub json_unused_externs: bool,
+}
+
 /// Parse the `--json` flag.
 ///
 /// The first value returned is how to render JSON diagnostics, and the second
 /// is whether or not artifact notifications are enabled.
-pub fn parse_json(matches: &getopts::Matches) -> (HumanReadableErrorType, bool) {
+pub fn parse_json(matches: &getopts::Matches) -> JsonConfig {
     let mut json_rendered: fn(ColorConfig) -> HumanReadableErrorType =
         HumanReadableErrorType::Default;
     let mut json_color = ColorConfig::Never;
     let mut json_artifact_notifications = false;
+    let mut json_unused_externs = false;
     for option in matches.opt_strs("json") {
         // For now conservatively forbid `--color` with `--json` since `--json`
         // won't actually be emitting any colors and anything colorized is
@@ -1280,6 +1241,7 @@ pub fn parse_json(matches: &getopts::Matches) -> (HumanReadableErrorType, bool)
                 "diagnostic-short" => json_rendered = HumanReadableErrorType::Short,
                 "diagnostic-rendered-ansi" => json_color = ColorConfig::Always,
                 "artifacts" => json_artifact_notifications = true,
+                "unused-externs" => json_unused_externs = true,
                 s => early_error(
                     ErrorOutputType::default(),
                     &format!("unknown `--json` option `{}`", s),
@@ -1287,7 +1249,12 @@ pub fn parse_json(matches: &getopts::Matches) -> (HumanReadableErrorType, bool)
             }
         }
     }
-    (json_rendered(json_color), json_artifact_notifications)
+
+    JsonConfig {
+        json_rendered: json_rendered(json_color),
+        json_artifact_notifications,
+        json_unused_externs,
+    }
 }
 
 /// Parses the `--error-format` flag.
@@ -1360,13 +1327,16 @@ pub fn parse_crate_edition(matches: &getopts::Matches) -> Edition {
     };
 
     if !edition.is_stable() && !nightly_options::is_unstable_enabled(matches) {
-        early_error(
-            ErrorOutputType::default(),
-            &format!(
-                "edition {} is unstable and only available with -Z unstable-options.",
-                edition,
-            ),
-        )
+        let is_nightly = nightly_options::match_is_nightly_build(matches);
+        let msg = if !is_nightly {
+            format!(
+                "the crate requires edition {}, but the latest edition supported by this Rust version is {}",
+                edition, LATEST_STABLE_EDITION
+            )
+        } else {
+            format!("edition {} is unstable and only available with -Z unstable-options", edition)
+        };
+        early_error(ErrorOutputType::default(), &msg)
     }
 
     edition
@@ -1865,7 +1835,8 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
 
     let edition = parse_crate_edition(matches);
 
-    let (json_rendered, json_artifact_notifications) = parse_json(matches);
+    let JsonConfig { json_rendered, json_artifact_notifications, json_unused_externs } =
+        parse_json(matches);
 
     let error_format = parse_error_format(matches, color, json_rendered);
 
@@ -1878,6 +1849,14 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
     let mut debugging_opts = build_debugging_options(matches, error_format);
     check_debug_option_stability(&debugging_opts, error_format, json_rendered);
 
+    if !debugging_opts.unstable_options && json_unused_externs {
+        early_error(
+            error_format,
+            "the `-Z unstable-options` flag must also be passed to enable \
+            the flag `--json=unused-externs`",
+        );
+    }
+
     let output_types = parse_output_types(&debugging_opts, matches, error_format);
 
     let mut cg = build_codegen_options(matches, error_format);
@@ -1916,7 +1895,9 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
         );
     }
 
-    if debugging_opts.instrument_coverage {
+    if debugging_opts.instrument_coverage.is_some()
+        && debugging_opts.instrument_coverage != Some(InstrumentCoverage::Off)
+    {
         if cg.profile_generate.enabled() || cg.profile_use.is_some() {
             early_error(
                 error_format,
@@ -1942,23 +1923,6 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
             }
             Some(SymbolManglingVersion::V0) => {}
         }
-
-        if debugging_opts.mir_opt_level > 1 {
-            // Functions inlined during MIR transform can, at best, make it impossible to
-            // effectively cover inlined functions, and, at worst, break coverage map generation
-            // during LLVM codegen. For example, function counter IDs are only unique within a
-            // function. Inlining after these counters are injected can produce duplicate counters,
-            // resulting in an invalid coverage map (and ICE); so this option combination is not
-            // allowed.
-            early_warn(
-                error_format,
-                &format!(
-                    "`-Z mir-opt-level={}` (or any level > 1) enables function inlining, which \
-                    is incompatible with `-Z instrument-coverage`. Inlining will be disabled.",
-                    debugging_opts.mir_opt_level,
-                ),
-            );
-        }
     }
 
     if let Ok(graphviz_font) = std::env::var("RUSTC_GRAPHVIZ_FONT") {
@@ -2053,6 +2017,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
         remap_path_prefix,
         edition,
         json_artifact_notifications,
+        json_unused_externs,
         pretty,
     }
 }
@@ -2077,6 +2042,7 @@ fn parse_pretty(
             ("hir,identified", true) => Hir(PpHirMode::Identified),
             ("hir,typed", true) => Hir(PpHirMode::Typed),
             ("hir-tree", true) => HirTree,
+            ("thir-tree", true) => ThirTree,
             ("mir", true) => Mir,
             ("mir-cfg", true) => MirCFG,
             _ => {
@@ -2268,6 +2234,8 @@ pub enum PpMode {
     Hir(PpHirMode),
     /// `-Zunpretty=hir-tree`
     HirTree,
+    /// `-Zunpretty=thir-tree`
+    ThirTree,
     /// `-Zunpretty=mir`
     Mir,
     /// `-Zunpretty=mir-cfg`
@@ -2285,6 +2253,7 @@ impl PpMode {
             | AstTree(PpAstTreeMode::Expanded)
             | Hir(_)
             | HirTree
+            | ThirTree
             | Mir
             | MirCFG => true,
         }
@@ -2292,7 +2261,7 @@ impl PpMode {
 
     pub fn needs_analysis(&self) -> bool {
         use PpMode::*;
-        matches!(*self, Mir | MirCFG)
+        matches!(*self, Mir | MirCFG | ThirTree)
     }
 }
 
@@ -2316,8 +2285,8 @@ impl PpMode {
 /// how the hash should be calculated when adding a new command-line argument.
 crate mod dep_tracking {
     use super::{
-        CFGuard, CrateType, DebugInfo, ErrorOutputType, LinkerPluginLto, LtoCli, OptLevel,
-        OutputTypes, Passes, SanitizerSet, SourceFileHashAlgorithm, SwitchWithOptPath,
+        CFGuard, CrateType, DebugInfo, ErrorOutputType, InstrumentCoverage, LinkerPluginLto,
+        LtoCli, OptLevel, OutputTypes, Passes, SourceFileHashAlgorithm, SwitchWithOptPath,
         SymbolManglingVersion, TrimmedDefPaths,
     };
     use crate::lint;
@@ -2326,7 +2295,7 @@ crate mod dep_tracking {
     use rustc_feature::UnstableFeatures;
     use rustc_span::edition::Edition;
     use rustc_target::spec::{CodeModel, MergeFunctions, PanicStrategy, RelocModel};
-    use rustc_target::spec::{RelroLevel, SplitDebuginfo, TargetTriple, TlsModel};
+    use rustc_target::spec::{RelroLevel, SanitizerSet, SplitDebuginfo, TargetTriple, TlsModel};
     use std::collections::hash_map::DefaultHasher;
     use std::collections::BTreeMap;
     use std::hash::Hash;
@@ -2370,6 +2339,7 @@ crate mod dep_tracking {
     impl_dep_tracking_hash_via_hash!(PathBuf);
     impl_dep_tracking_hash_via_hash!(lint::Level);
     impl_dep_tracking_hash_via_hash!(Option<bool>);
+    impl_dep_tracking_hash_via_hash!(Option<u32>);
     impl_dep_tracking_hash_via_hash!(Option<usize>);
     impl_dep_tracking_hash_via_hash!(Option<NonZeroUsize>);
     impl_dep_tracking_hash_via_hash!(Option<String>);
@@ -2382,6 +2352,7 @@ crate mod dep_tracking {
     impl_dep_tracking_hash_via_hash!(Option<WasiExecModel>);
     impl_dep_tracking_hash_via_hash!(Option<PanicStrategy>);
     impl_dep_tracking_hash_via_hash!(Option<RelroLevel>);
+    impl_dep_tracking_hash_via_hash!(Option<InstrumentCoverage>);
     impl_dep_tracking_hash_via_hash!(Option<lint::Level>);
     impl_dep_tracking_hash_via_hash!(Option<PathBuf>);
     impl_dep_tracking_hash_via_hash!(CrateType);
@@ -2443,7 +2414,7 @@ crate mod dep_tracking {
     }
 
     // This is a stable hash because BTreeMap is a sorted container
-    pub fn stable_hash(
+    crate fn stable_hash(
         sub_hashes: BTreeMap<&'static str, &dyn DepTrackingHash>,
         hasher: &mut DefaultHasher,
         error_format: ErrorOutputType,
diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs
index d002f597391..7971f7ef9ef 100644
--- a/compiler/rustc_session/src/lib.rs
+++ b/compiler/rustc_session/src/lib.rs
@@ -1,10 +1,9 @@
 #![feature(crate_visibility_modifier)]
 #![feature(once_cell)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
+#![recursion_limit = "256"]
 
 #[macro_use]
-extern crate bitflags;
-#[macro_use]
 extern crate rustc_macros;
 
 pub mod cgu_reuse_tracker;
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 6e7d39547a1..fd26f50da5a 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -5,7 +5,7 @@ use crate::lint;
 use crate::search_paths::SearchPath;
 use crate::utils::NativeLibKind;
 
-use rustc_target::spec::{CodeModel, LinkerFlavor, MergeFunctions, PanicStrategy};
+use rustc_target::spec::{CodeModel, LinkerFlavor, MergeFunctions, PanicStrategy, SanitizerSet};
 use rustc_target::spec::{RelocModel, RelroLevel, SplitDebuginfo, TargetTriple, TlsModel};
 
 use rustc_feature::UnstableFeatures;
@@ -147,6 +147,9 @@ top_level_options!(
         // by the compiler.
         json_artifact_notifications: bool [TRACKED],
 
+        // `true` if we're emitting a JSON blob containing the unused externs
+        json_unused_externs: bool [UNTRACKED],
+
         pretty: Option<PpMode> [UNTRACKED],
     }
 );
@@ -248,9 +251,9 @@ macro_rules! options {
         pub const parse_list: &str = "a space-separated list of strings";
         pub const parse_opt_list: &str = parse_list;
         pub const parse_opt_comma_list: &str = "a comma-separated list of strings";
-        pub const parse_uint: &str = "a number";
-        pub const parse_opt_uint: &str = parse_uint;
-        pub const parse_threads: &str = parse_uint;
+        pub const parse_number: &str = "a number";
+        pub const parse_opt_number: &str = parse_number;
+        pub const parse_threads: &str = parse_number;
         pub const parse_passes: &str = "a space-separated list of passes, or `all`";
         pub const parse_panic_strategy: &str = "either `unwind` or `abort`";
         pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
@@ -262,6 +265,7 @@ macro_rules! options {
         pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavor::one_of();
         pub const parse_optimization_fuel: &str = "crate=integer";
         pub const parse_mir_spanview: &str = "`statement` (default), `terminator`, or `block`";
+        pub const parse_instrument_coverage: &str = "`all` (default), `except-unused-generics`, `except-unused-functions`, or `off`";
         pub const parse_unpretty: &str = "`string` or `string=string`";
         pub const parse_treat_err_as_bug: &str = "either no value or a number bigger than 0";
         pub const parse_lto: &str =
@@ -413,16 +417,16 @@ macro_rules! options {
             }
         }
 
-        /// Use this for any uint option that has a static default.
-        fn parse_uint(slot: &mut usize, v: Option<&str>) -> bool {
+        /// Use this for any numeric option that has a static default.
+        fn parse_number<T: Copy + FromStr>(slot: &mut T, v: Option<&str>) -> bool {
             match v.and_then(|s| s.parse().ok()) {
                 Some(i) => { *slot = i; true },
                 None => false
             }
         }
 
-        /// Use this for any uint option that lacks a static default.
-        fn parse_opt_uint(slot: &mut Option<usize>, v: Option<&str>) -> bool {
+        /// Use this for any numeric option that lacks a static default.
+        fn parse_opt_number<T: Copy + FromStr>(slot: &mut Option<T>, v: Option<&str>) -> bool {
             match v {
                 Some(s) => { *slot = s.parse().ok(); slot.is_some() }
                 None => false
@@ -592,6 +596,41 @@ macro_rules! options {
             true
         }
 
+        fn parse_instrument_coverage(slot: &mut Option<InstrumentCoverage>, v: Option<&str>) -> bool {
+            if v.is_some() {
+                let mut bool_arg = None;
+                if parse_opt_bool(&mut bool_arg, v) {
+                    *slot = if bool_arg.unwrap() {
+                        Some(InstrumentCoverage::All)
+                    } else {
+                        None
+                    };
+                    return true
+                }
+            }
+
+            let v = match v {
+                None => {
+                    *slot = Some(InstrumentCoverage::All);
+                    return true;
+                }
+                Some(v) => v,
+            };
+
+            *slot = Some(match v {
+                "all" => InstrumentCoverage::All,
+                "except-unused-generics" | "except_unused_generics" => {
+                    InstrumentCoverage::ExceptUnusedGenerics
+                }
+                "except-unused-functions" | "except_unused_functions" => {
+                    InstrumentCoverage::ExceptUnusedFunctions
+                }
+                "off" | "no" | "n" | "false" | "0" => InstrumentCoverage::Off,
+                _ => return false,
+            });
+            true
+        }
+
         fn parse_treat_err_as_bug(slot: &mut Option<NonZeroUsize>, v: Option<&str>) -> bool {
             match v {
                 Some(s) => { *slot = s.parse().ok(); slot.is_some() }
@@ -748,13 +787,13 @@ options! {CodegenOptions, CodegenSetter, basic_codegen_options,
         "this option is deprecated and does nothing"),
     code_model: Option<CodeModel> = (None, parse_code_model, [TRACKED],
         "choose the code model to use (`rustc --print code-models` for details)"),
-    codegen_units: Option<usize> = (None, parse_opt_uint, [UNTRACKED],
+    codegen_units: Option<usize> = (None, parse_opt_number, [UNTRACKED],
         "divide crate into N units to optimize in parallel"),
     control_flow_guard: CFGuard = (CFGuard::Disabled, parse_cfguard, [TRACKED],
         "use Windows Control Flow Guard (default: no)"),
     debug_assertions: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "explicitly enable the `cfg(debug_assertions)` directive"),
-    debuginfo: usize = (0, parse_uint, [TRACKED],
+    debuginfo: usize = (0, parse_number, [TRACKED],
         "debug info emission level (0 = no debug info, 1 = line tables only, \
         2 = full debug info with variable and type information; default: 0)"),
     default_linker_libraries: bool = (false, parse_bool, [UNTRACKED],
@@ -769,13 +808,13 @@ options! {CodegenOptions, CodegenSetter, basic_codegen_options,
         "force use of unwind tables"),
     incremental: Option<String> = (None, parse_opt_string, [UNTRACKED],
         "enable incremental compilation"),
-    inline_threshold: Option<usize> = (None, parse_opt_uint, [TRACKED],
+    inline_threshold: Option<u32> = (None, parse_opt_number, [TRACKED],
         "set the threshold for inlining a function"),
     link_arg: (/* redirected to link_args */) = ((), parse_string_push, [UNTRACKED],
         "a single extra argument to append to the linker invocation (can be used several times)"),
     link_args: Vec<String> = (Vec::new(), parse_list, [UNTRACKED],
         "extra arguments to append to the linker invocation (space separated)"),
-    link_dead_code: Option<bool> = (None, parse_opt_bool, [UNTRACKED],
+    link_dead_code: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "keep dead code at link time (useful for code coverage) (default: no)"),
     link_self_contained: Option<bool> = (None, parse_opt_bool, [UNTRACKED],
         "control whether to link Rust provided C objects/libraries or rely
@@ -870,8 +909,6 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         (default: no)"),
     borrowck: String = ("migrate".to_string(), parse_string, [UNTRACKED],
         "select which borrowck is used (`mir` or `migrate`) (default: `migrate`)"),
-    borrowck_stats: bool = (false, parse_bool, [UNTRACKED],
-        "gather borrowck statistics (default: no)"),
     cgu_partitioning_strategy: Option<String> = (None, parse_opt_string, [TRACKED],
         "the codegen unit partitioning strategy to use"),
     chalk: bool = (false, parse_bool, [TRACKED],
@@ -959,24 +996,22 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "verify incr. comp. hashes of green query instances (default: no)"),
     inline_mir: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "enable MIR inlining (default: no)"),
-    inline_mir_threshold: Option<usize> = (None, parse_opt_uint, [TRACKED],
+    inline_mir_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
         "a default MIR inlining threshold (default: 50)"),
-    inline_mir_hint_threshold: Option<usize> = (None, parse_opt_uint, [TRACKED],
+    inline_mir_hint_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
         "inlining threshold for functions with inline hint (default: 100)"),
     inline_in_all_cgus: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "control whether `#[inline]` functions are in all CGUs"),
     input_stats: bool = (false, parse_bool, [UNTRACKED],
         "gather statistics about the input (default: no)"),
-    insert_sideeffect: bool = (false, parse_bool, [TRACKED],
-        "fix undefined behavior when a thread doesn't eventually make progress \
-        (such as entering an empty infinite loop) by inserting llvm.sideeffect \
-        (default: no)"),
-    instrument_coverage: bool = (false, parse_bool, [TRACKED],
+    instrument_coverage: Option<InstrumentCoverage> = (None, parse_instrument_coverage, [TRACKED],
         "instrument the generated code to support LLVM source-based code coverage \
         reports (note, the compiler build config must include `profiler = true`, \
         and is mutually exclusive with `-C profile-generate`/`-C profile-use`); \
         implies `-Z symbol-mangling-version=v0`; disables/overrides some Rust \
-        optimizations (default: no)"),
+        optimizations. Optional values are: `=all` (default coverage), \
+        `=except-unused-generics`, `=except-unused-functions`, or `=off` \
+        (default: instrument-coverage=off)"),
     instrument_mcount: bool = (false, parse_bool, [TRACKED],
         "insert function instrument code for mcount-based tracing (default: no)"),
     keep_hygiene_data: bool = (false, parse_bool, [UNTRACKED],
@@ -999,10 +1034,10 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
     mir_emit_retag: bool = (false, parse_bool, [TRACKED],
         "emit Retagging MIR statements, interpreted e.g., by miri; implies -Zmir-opt-level=0 \
         (default: no)"),
-    mir_opt_level: usize = (1, parse_uint, [TRACKED],
-        "MIR optimization level (0-3; default: 1)"),
-    mutable_noalias: bool = (false, parse_bool, [TRACKED],
-        "emit noalias metadata for mutable references (default: no)"),
+    mir_opt_level: Option<usize> = (None, parse_opt_number, [TRACKED],
+        "MIR optimization level (0-4; default: 1 in non optimized builds and 2 in optimized builds)"),
+    mutable_noalias: Option<bool> = (None, parse_opt_bool, [TRACKED],
+        "emit noalias metadata for mutable references (default: yes for LLVM >= 12, otherwise no)"),
     new_llvm_pass_manager: bool = (false, parse_bool, [TRACKED],
         "use new LLVM pass manager (default: no)"),
     nll_facts: bool = (false, parse_bool, [UNTRACKED],
@@ -1120,7 +1155,7 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "which mangling version to use for symbol names ('legacy' (default) or 'v0')"),
     teach: bool = (false, parse_bool, [TRACKED],
         "show extended diagnostic help (default: no)"),
-    terminal_width: Option<usize> = (None, parse_opt_uint, [UNTRACKED],
+    terminal_width: Option<usize> = (None, parse_opt_number, [UNTRACKED],
         "set the current terminal width"),
     tune_cpu: Option<String> = (None, parse_opt_string, [TRACKED],
         "select processor to schedule for (`rustc --print target-cpus` for details)"),
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 81b38347414..65d5d96aba1 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -13,7 +13,6 @@ use rustc_span::hygiene::ExpnId;
 use rustc_span::source_map::{FilePathMapping, SourceMap};
 use rustc_span::{MultiSpan, Span, Symbol};
 
-use std::path::PathBuf;
 use std::str;
 
 /// The set of keys (and, optionally, values) that define the compilation
@@ -122,8 +121,6 @@ pub struct ParseSess {
     pub missing_fragment_specifiers: Lock<FxHashMap<Span, NodeId>>,
     /// Places where raw identifiers were used. This is used for feature-gating raw identifiers.
     pub raw_identifier_spans: Lock<Vec<Span>>,
-    /// Used to determine and report recursive module inclusions.
-    pub included_mod_stack: Lock<Vec<PathBuf>>,
     source_map: Lrc<SourceMap>,
     pub buffered_lints: Lock<Vec<BufferedEarlyLint>>,
     /// Contains the spans of block expressions that could have been incomplete based on the
@@ -143,6 +140,7 @@ pub struct ParseSess {
 }
 
 impl ParseSess {
+    /// Used for testing.
     pub fn new(file_path_mapping: FilePathMapping) -> Self {
         let sm = Lrc::new(SourceMap::new(file_path_mapping));
         let handler = Handler::with_tty_emitter(ColorConfig::Auto, true, None, Some(sm.clone()));
@@ -157,7 +155,6 @@ impl ParseSess {
             edition: ExpnId::root().expn_data().edition,
             missing_fragment_specifiers: Default::default(),
             raw_identifier_spans: Lock::new(Vec::new()),
-            included_mod_stack: Lock::new(vec![]),
             source_map,
             buffered_lints: Lock::new(vec![]),
             ambiguous_block_expr_parse: Lock::new(FxHashMap::default()),
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 08a7447008a..e3ea3a11c4d 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -1,14 +1,13 @@
 use crate::cgu_reuse_tracker::CguReuseTracker;
 use crate::code_stats::CodeStats;
 pub use crate::code_stats::{DataTypeKind, FieldInfo, SizeKind, VariantInfo};
-use crate::config::{self, CrateType, OutputType, PrintRequest, SanitizerSet, SwitchWithOptPath};
+use crate::config::{self, CrateType, OutputType, PrintRequest, SwitchWithOptPath};
 use crate::filesearch;
 use crate::lint::{self, LintId};
 use crate::parse::ParseSess;
 use crate::search_paths::{PathKind, SearchPath};
 
 pub use rustc_ast::attr::MarkedAttrs;
-pub use rustc_ast::crate_disambiguator::CrateDisambiguator;
 pub use rustc_ast::Attribute;
 use rustc_data_structures::flock;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
@@ -21,14 +20,15 @@ use rustc_errors::annotate_snippet_emitter_writer::AnnotateSnippetEmitterWriter;
 use rustc_errors::emitter::{Emitter, EmitterWriter, HumanReadableErrorType};
 use rustc_errors::json::JsonEmitter;
 use rustc_errors::registry::Registry;
-use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, DiagnosticId, ErrorReported};
+use rustc_errors::{Diagnostic, DiagnosticBuilder, DiagnosticId, ErrorReported};
 use rustc_lint_defs::FutureBreakage;
+pub use rustc_span::crate_disambiguator::CrateDisambiguator;
 use rustc_span::edition::Edition;
 use rustc_span::source_map::{FileLoader, MultiSpan, RealFileLoader, SourceMap, Span};
 use rustc_span::{sym, SourceFileHashAlgorithm, Symbol};
 use rustc_target::asm::InlineAsmArch;
 use rustc_target::spec::{CodeModel, PanicStrategy, RelocModel, RelroLevel};
-use rustc_target::spec::{SplitDebuginfo, Target, TargetTriple, TlsModel};
+use rustc_target::spec::{SanitizerSet, SplitDebuginfo, Target, TargetTriple, TlsModel};
 
 use std::cell::{self, RefCell};
 use std::env;
@@ -241,8 +241,7 @@ pub struct PerfStats {
 enum DiagnosticBuilderMethod {
     Note,
     SpanNote,
-    SpanSuggestion(String), // suggestion
-                            // Add more variants as needed to support one-time diagnostics.
+    // Add more variants as needed to support one-time diagnostics.
 }
 
 /// Trait implemented by error types. This should not be implemented manually. Instead, use
@@ -551,15 +550,6 @@ impl Session {
                     let span = span_maybe.expect("`span_note` needs a span");
                     diag_builder.span_note(span, message);
                 }
-                DiagnosticBuilderMethod::SpanSuggestion(suggestion) => {
-                    let span = span_maybe.expect("`span_suggestion_*` needs a span");
-                    diag_builder.span_suggestion(
-                        span,
-                        message,
-                        suggestion,
-                        Applicability::Unspecified,
-                    );
-                }
             }
         }
     }
@@ -589,23 +579,6 @@ impl Session {
         self.diag_once(diag_builder, DiagnosticBuilderMethod::Note, msg_id, message, None);
     }
 
-    pub fn diag_span_suggestion_once<'a, 'b>(
-        &'a self,
-        diag_builder: &'b mut DiagnosticBuilder<'a>,
-        msg_id: DiagnosticMessageId,
-        span: Span,
-        message: &str,
-        suggestion: String,
-    ) {
-        self.diag_once(
-            diag_builder,
-            DiagnosticBuilderMethod::SpanSuggestion(suggestion),
-            msg_id,
-            message,
-            Some(span),
-        );
-    }
-
     #[inline]
     pub fn source_map(&self) -> &SourceMap {
         self.parse_sess.source_map()
@@ -631,15 +604,18 @@ impl Session {
     pub fn verify_llvm_ir(&self) -> bool {
         self.opts.debugging_opts.verify_llvm_ir || option_env!("RUSTC_VERIFY_LLVM_IR").is_some()
     }
-    pub fn borrowck_stats(&self) -> bool {
-        self.opts.debugging_opts.borrowck_stats
-    }
     pub fn print_llvm_passes(&self) -> bool {
         self.opts.debugging_opts.print_llvm_passes
     }
     pub fn binary_dep_depinfo(&self) -> bool {
         self.opts.debugging_opts.binary_dep_depinfo
     }
+    pub fn mir_opt_level(&self) -> usize {
+        self.opts
+            .debugging_opts
+            .mir_opt_level
+            .unwrap_or_else(|| if self.opts.optimize != config::OptLevel::No { 2 } else { 1 })
+    }
 
     /// Gets the features enabled for the current compilation session.
     /// DO NOT USE THIS METHOD if there is a TyCtxt available, as it circumvents
@@ -831,8 +807,11 @@ impl Session {
         // This is used to control the emission of the `uwtable` attribute on
         // LLVM functions.
         //
-        // At the very least, unwind tables are needed when compiling with
-        // `-C panic=unwind`.
+        // Unwind tables are needed when compiling with `-C panic=unwind`, but
+        // LLVM won't omit unwind tables unless the function is also marked as
+        // `nounwind`, so users are allowed to disable `uwtable` emission.
+        // Historically rustc always emits `uwtable` attributes by default, so
+        // even they can be disabled, they're still emitted by default.
         //
         // On some targets (including windows), however, exceptions include
         // other events such as illegal instructions, segfaults, etc. This means
@@ -845,13 +824,10 @@ impl Session {
         // If a target requires unwind tables, then they must be emitted.
         // Otherwise, we can defer to the `-C force-unwind-tables=<yes/no>`
         // value, if it is provided, or disable them, if not.
-        if self.panic_strategy() == PanicStrategy::Unwind {
-            true
-        } else if self.target.requires_uwtable {
-            true
-        } else {
-            self.opts.cg.force_unwind_tables.unwrap_or(false)
-        }
+        self.target.requires_uwtable
+            || self.opts.cg.force_unwind_tables.unwrap_or(
+                self.panic_strategy() == PanicStrategy::Unwind || self.target.default_uwtable,
+            )
     }
 
     /// Returns the symbol name for the registrar function,
@@ -884,22 +860,6 @@ impl Session {
         )
     }
 
-    pub fn set_incr_session_load_dep_graph(&self, load: bool) {
-        let mut incr_comp_session = self.incr_comp_session.borrow_mut();
-
-        if let IncrCompSession::Active { ref mut load_dep_graph, .. } = *incr_comp_session {
-            *load_dep_graph = load;
-        }
-    }
-
-    pub fn incr_session_load_dep_graph(&self) -> bool {
-        let incr_comp_session = self.incr_comp_session.borrow();
-        match *incr_comp_session {
-            IncrCompSession::Active { load_dep_graph, .. } => load_dep_graph,
-            _ => false,
-        }
-    }
-
     pub fn init_incr_comp_session(
         &self,
         session_dir: PathBuf,
@@ -1137,6 +1097,21 @@ impl Session {
         self.opts.cg.link_dead_code.unwrap_or(false)
     }
 
+    pub fn instrument_coverage(&self) -> bool {
+        self.opts.debugging_opts.instrument_coverage.unwrap_or(config::InstrumentCoverage::Off)
+            != config::InstrumentCoverage::Off
+    }
+
+    pub fn instrument_coverage_except_unused_generics(&self) -> bool {
+        self.opts.debugging_opts.instrument_coverage.unwrap_or(config::InstrumentCoverage::Off)
+            == config::InstrumentCoverage::ExceptUnusedGenerics
+    }
+
+    pub fn instrument_coverage_except_unused_functions(&self) -> bool {
+        self.opts.debugging_opts.instrument_coverage.unwrap_or(config::InstrumentCoverage::Off)
+            == config::InstrumentCoverage::ExceptUnusedFunctions
+    }
+
     pub fn mark_attr_known(&self, attr: &Attribute) {
         self.known_attrs.lock().mark(attr)
     }
@@ -1509,13 +1484,6 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
 
     // Unwind tables cannot be disabled if the target requires them.
     if let Some(include_uwtables) = sess.opts.cg.force_unwind_tables {
-        if sess.panic_strategy() == PanicStrategy::Unwind && !include_uwtables {
-            sess.err(
-                "panic=unwind requires unwind tables, they cannot be disabled \
-                     with `-C force-unwind-tables=no`.",
-            );
-        }
-
         if sess.target.requires_uwtable && !include_uwtables {
             sess.err(
                 "target requires unwind tables, they cannot be disabled with \
@@ -1543,59 +1511,22 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
         );
     }
 
-    const ASAN_SUPPORTED_TARGETS: &[&str] = &[
-        "aarch64-apple-darwin",
-        "aarch64-fuchsia",
-        "aarch64-unknown-linux-gnu",
-        "x86_64-apple-darwin",
-        "x86_64-fuchsia",
-        "x86_64-unknown-freebsd",
-        "x86_64-unknown-linux-gnu",
-    ];
-    const LSAN_SUPPORTED_TARGETS: &[&str] = &[
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "x86_64-apple-darwin",
-        "x86_64-unknown-linux-gnu",
-    ];
-    const MSAN_SUPPORTED_TARGETS: &[&str] =
-        &["aarch64-unknown-linux-gnu", "x86_64-unknown-freebsd", "x86_64-unknown-linux-gnu"];
-    const TSAN_SUPPORTED_TARGETS: &[&str] = &[
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "x86_64-apple-darwin",
-        "x86_64-unknown-freebsd",
-        "x86_64-unknown-linux-gnu",
-    ];
-    const HWASAN_SUPPORTED_TARGETS: &[&str] =
-        &["aarch64-linux-android", "aarch64-unknown-linux-gnu"];
-
-    // Sanitizers can only be used on some tested platforms.
-    for s in sess.opts.debugging_opts.sanitizer {
-        let supported_targets = match s {
-            SanitizerSet::ADDRESS => ASAN_SUPPORTED_TARGETS,
-            SanitizerSet::LEAK => LSAN_SUPPORTED_TARGETS,
-            SanitizerSet::MEMORY => MSAN_SUPPORTED_TARGETS,
-            SanitizerSet::THREAD => TSAN_SUPPORTED_TARGETS,
-            SanitizerSet::HWADDRESS => HWASAN_SUPPORTED_TARGETS,
-            _ => panic!("unrecognized sanitizer {}", s),
-        };
-        if !supported_targets.contains(&&*sess.opts.target_triple.triple()) {
-            sess.err(&format!(
-                "`-Zsanitizer={}` only works with targets: {}",
-                s,
-                supported_targets.join(", ")
-            ));
-        }
-        let conflicting = sess.opts.debugging_opts.sanitizer - s;
-        if !conflicting.is_empty() {
-            sess.err(&format!(
-                "`-Zsanitizer={}` is incompatible with `-Zsanitizer={}`",
-                s, conflicting,
-            ));
-            // Don't report additional errors.
-            break;
-        }
+    // Sanitizers can only be used on platforms that we know have working sanitizer codegen.
+    let supported_sanitizers = sess.target.options.supported_sanitizers;
+    let unsupported_sanitizers = sess.opts.debugging_opts.sanitizer - supported_sanitizers;
+    match unsupported_sanitizers.into_iter().count() {
+        0 => {}
+        1 => sess
+            .err(&format!("{} sanitizer is not supported for this target", unsupported_sanitizers)),
+        _ => sess.err(&format!(
+            "{} sanitizers are not supported for this target",
+            unsupported_sanitizers
+        )),
+    }
+    // Cannot mix and match sanitizers.
+    let mut sanitizer_iter = sess.opts.debugging_opts.sanitizer.into_iter();
+    if let (Some(first), Some(second)) = (sanitizer_iter.next(), sanitizer_iter.next()) {
+        sess.err(&format!("`-Zsanitizer={}` is incompatible with `-Zsanitizer={}`", first, second));
     }
 }
 
diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs
index f3d33309124..e9d597d1ba6 100644
--- a/compiler/rustc_session/src/utils.rs
+++ b/compiler/rustc_session/src/utils.rs
@@ -1,7 +1,13 @@
+use crate::parse::ParseSess;
 use crate::session::Session;
+use rustc_ast::token::{self, DelimToken, Nonterminal, Token};
+use rustc_ast::tokenstream::CanSynthesizeMissingTokens;
+use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use rustc_data_structures::profiling::VerboseTimingGuard;
 use std::path::{Path, PathBuf};
 
+pub type NtToTokenstream = fn(&Nonterminal, &ParseSess, CanSynthesizeMissingTokens) -> TokenStream;
+
 impl Session {
     pub fn timer<'a>(&'a self, what: &'static str) -> VerboseTimingGuard<'a> {
         self.prof.verbose_generic_activity(what)
@@ -53,3 +59,52 @@ impl CanonicalizedPath {
         &self.original
     }
 }
+
+// FIXME: Find a better spot for this - it needs to be accessible from `rustc_ast_lowering`,
+// and needs to access `ParseSess
+pub struct FlattenNonterminals<'a> {
+    pub parse_sess: &'a ParseSess,
+    pub synthesize_tokens: CanSynthesizeMissingTokens,
+    pub nt_to_tokenstream: NtToTokenstream,
+}
+
+impl<'a> FlattenNonterminals<'a> {
+    pub fn process_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
+        fn can_skip(stream: &TokenStream) -> bool {
+            stream.trees().all(|tree| match tree {
+                TokenTree::Token(token) => !matches!(token.kind, token::Interpolated(_)),
+                TokenTree::Delimited(_, _, inner) => can_skip(&inner),
+            })
+        }
+
+        if can_skip(&tokens) {
+            return tokens;
+        }
+
+        tokens.into_trees().flat_map(|tree| self.process_token_tree(tree).into_trees()).collect()
+    }
+
+    pub fn process_token_tree(&mut self, tree: TokenTree) -> TokenStream {
+        match tree {
+            TokenTree::Token(token) => self.process_token(token),
+            TokenTree::Delimited(span, delim, tts) => {
+                TokenTree::Delimited(span, delim, self.process_token_stream(tts)).into()
+            }
+        }
+    }
+
+    pub fn process_token(&mut self, token: Token) -> TokenStream {
+        match token.kind {
+            token::Interpolated(nt) => {
+                let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
+                TokenTree::Delimited(
+                    DelimSpan::from_single(token.span),
+                    DelimToken::NoDelim,
+                    self.process_token_stream(tts),
+                )
+                .into()
+            }
+            _ => TokenTree::Token(token).into(),
+        }
+    }
+}
diff --git a/compiler/rustc_ast/src/crate_disambiguator.rs b/compiler/rustc_span/src/crate_disambiguator.rs
index bd7d8516714..bd7d8516714 100644
--- a/compiler/rustc_ast/src/crate_disambiguator.rs
+++ b/compiler/rustc_span/src/crate_disambiguator.rs
diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs
index 70e9526f626..95bb0ad7ba2 100644
--- a/compiler/rustc_span/src/def_id.rs
+++ b/compiler/rustc_span/src/def_id.rs
@@ -1,3 +1,4 @@
+use crate::crate_disambiguator::CrateDisambiguator;
 use crate::HashStableContext;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -105,10 +106,74 @@ impl ::std::fmt::Debug for CrateNum {
     }
 }
 
+/// A `DefPathHash` is a fixed-size representation of a `DefPath` that is
+/// stable across crate and compilation session boundaries. It consists of two
+/// separate 64-bit hashes. The first uniquely identifies the crate this
+/// `DefPathHash` originates from (see [StableCrateId]), and the second
+/// uniquely identifies the corresponding `DefPath` within that crate. Together
+/// they form a unique identifier within an entire crate graph.
+///
+/// There is a very small chance of hash collisions, which would mean that two
+/// different `DefPath`s map to the same `DefPathHash`. Proceeding compilation
+/// with such a hash collision would very probably lead to an ICE, and in the
+/// worst case lead to a silent mis-compilation. The compiler therefore actively
+/// and exhaustively checks for such hash collisions and aborts compilation if
+/// it finds one.
+///
+/// `DefPathHash` uses 64-bit hashes for both the crate-id part and the
+/// crate-internal part, even though it is likely that there are many more
+/// `LocalDefId`s in a single crate than there are individual crates in a crate
+/// graph. Since we use the same number of bits in both cases, the collision
+/// probability for the crate-local part will be quite a bit higher (though
+/// still very small).
+///
+/// This imbalance is not by accident: A hash collision in the
+/// crate-local part of a `DefPathHash` will be detected and reported while
+/// compiling the crate in question. Such a collision does not depend on
+/// outside factors and can be easily fixed by the crate maintainer (e.g. by
+/// renaming the item in question or by bumping the crate version in a harmless
+/// way).
+///
+/// A collision between crate-id hashes on the other hand is harder to fix
+/// because it depends on the set of crates in the entire crate graph of a
+/// compilation session. Again, using the same crate with a different version
+/// number would fix the issue with a high probability -- but that might be
+/// easier said then done if the crates in questions are dependencies of
+/// third-party crates.
+///
+/// That being said, given a high quality hash function, the collision
+/// probabilities in question are very small. For example, for a big crate like
+/// `rustc_middle` (with ~50000 `LocalDefId`s as of the time of writing) there
+/// is a probability of roughly 1 in 14,750,000,000 of a crate-internal
+/// collision occurring. For a big crate graph with 1000 crates in it, there is
+/// a probability of 1 in 36,890,000,000,000 of a `StableCrateId` collision.
 #[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Debug)]
 #[derive(HashStable_Generic, Encodable, Decodable)]
 pub struct DefPathHash(pub Fingerprint);
 
+impl DefPathHash {
+    /// Returns the [StableCrateId] identifying the crate this [DefPathHash]
+    /// originates from.
+    #[inline]
+    pub fn stable_crate_id(&self) -> StableCrateId {
+        StableCrateId(self.0.as_value().0)
+    }
+
+    /// Returns the crate-local part of the [DefPathHash].
+    ///
+    /// Used for tests.
+    #[inline]
+    pub fn local_hash(&self) -> u64 {
+        self.0.as_value().1
+    }
+
+    /// Builds a new [DefPathHash] with the given [StableCrateId] and
+    /// `local_hash`, where `local_hash` must be unique within its crate.
+    pub fn new(stable_crate_id: StableCrateId, local_hash: u64) -> DefPathHash {
+        DefPathHash(Fingerprint::new(stable_crate_id.0, local_hash))
+    }
+}
+
 impl Borrow<Fingerprint> for DefPathHash {
     #[inline]
     fn borrow(&self) -> &Fingerprint {
@@ -116,6 +181,30 @@ impl Borrow<Fingerprint> for DefPathHash {
     }
 }
 
+/// A [StableCrateId] is a 64 bit hash of `(crate-name, crate-disambiguator)`. It
+/// is to [CrateNum] what [DefPathHash] is to [DefId]. It is stable across
+/// compilation sessions.
+///
+/// Since the ID is a hash value there is a (very small) chance that two crates
+/// end up with the same [StableCrateId]. The compiler will check for such
+/// collisions when loading crates and abort compilation in order to avoid
+/// further trouble.
+#[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Encodable, Decodable)]
+pub struct StableCrateId(u64);
+
+impl StableCrateId {
+    /// Computes the stable ID for a crate with the given name and
+    /// disambiguator.
+    pub fn new(crate_name: &str, crate_disambiguator: CrateDisambiguator) -> StableCrateId {
+        use std::hash::Hash;
+
+        let mut hasher = StableHasher::new();
+        crate_name.hash(&mut hasher);
+        crate_disambiguator.hash(&mut hasher);
+        StableCrateId(hasher.finish())
+    }
+}
+
 rustc_index::newtype_index! {
     /// A DefIndex is an index into the hir-map for a crate, identifying a
     /// particular definition. It should really be considered an interned
diff --git a/compiler/rustc_span/src/edition.rs b/compiler/rustc_span/src/edition.rs
index a9200dd7dfd..8544acd6d05 100644
--- a/compiler/rustc_span/src/edition.rs
+++ b/compiler/rustc_span/src/edition.rs
@@ -20,7 +20,7 @@ pub enum Edition {
     Edition2015,
     /// The 2018 edition
     Edition2018,
-    /// The 2021 ediiton
+    /// The 2021 edition
     Edition2021,
 }
 
diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs
index 4ccf657335f..f75fe22767f 100644
--- a/compiler/rustc_span/src/hygiene.rs
+++ b/compiler/rustc_span/src/hygiene.rs
@@ -118,7 +118,8 @@ impl ExpnId {
         HygieneData::with(|data| {
             let old_expn_data = &mut data.expn_data[self.0 as usize];
             assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID");
-            expn_data.orig_id.replace(self.as_u32()).expect_none("orig_id should be None");
+            assert_eq!(expn_data.orig_id, None);
+            expn_data.orig_id = Some(self.as_u32());
             *old_expn_data = Some(expn_data);
         });
         update_disambiguator(self)
@@ -202,7 +203,8 @@ impl HygieneData {
     fn fresh_expn(&mut self, mut expn_data: Option<ExpnData>) -> ExpnId {
         let raw_id = self.expn_data.len() as u32;
         if let Some(data) = expn_data.as_mut() {
-            data.orig_id.replace(raw_id).expect_none("orig_id should be None");
+            assert_eq!(data.orig_id, None);
+            data.orig_id = Some(raw_id);
         }
         self.expn_data.push(expn_data);
         ExpnId(raw_id)
@@ -411,7 +413,7 @@ pub fn update_dollar_crate_names(mut get_name: impl FnMut(SyntaxContext) -> Symb
     let names: Vec<_> =
         range_to_update.clone().map(|idx| get_name(SyntaxContext::from_u32(idx as u32))).collect();
     HygieneData::with(|data| {
-        range_to_update.zip(names.into_iter()).for_each(|(idx, name)| {
+        range_to_update.zip(names).for_each(|(idx, name)| {
             data.syntax_context_data[idx].dollar_crate_name = name;
         })
     })
@@ -746,7 +748,7 @@ pub struct ExpnData {
 
     /// Used to force two `ExpnData`s to have different `Fingerprint`s.
     /// Due to macro expansion, it's possible to end up with two `ExpnId`s
-    /// that have identical `ExpnData`s. This violates the constract of `HashStable`
+    /// that have identical `ExpnData`s. This violates the contract of `HashStable`
     /// - the two `ExpnId`s are not equal, but their `Fingerprint`s are equal
     /// (since the numerical `ExpnId` value is not considered by the `HashStable`
     /// implementation).
@@ -1174,11 +1176,7 @@ pub fn decode_syntax_context<
     Ok(new_ctxt)
 }
 
-pub fn num_syntax_ctxts() -> usize {
-    HygieneData::with(|data| data.syntax_context_data.len())
-}
-
-pub fn for_all_ctxts_in<E, F: FnMut((u32, SyntaxContext, &SyntaxContextData)) -> Result<(), E>>(
+fn for_all_ctxts_in<E, F: FnMut((u32, SyntaxContext, &SyntaxContextData)) -> Result<(), E>>(
     ctxts: impl Iterator<Item = SyntaxContext>,
     mut f: F,
 ) -> Result<(), E> {
@@ -1191,7 +1189,7 @@ pub fn for_all_ctxts_in<E, F: FnMut((u32, SyntaxContext, &SyntaxContextData)) ->
     Ok(())
 }
 
-pub fn for_all_expns_in<E, F: FnMut(u32, ExpnId, &ExpnData) -> Result<(), E>>(
+fn for_all_expns_in<E, F: FnMut(u32, ExpnId, &ExpnData) -> Result<(), E>>(
     expns: impl Iterator<Item = ExpnId>,
     mut f: F,
 ) -> Result<(), E> {
@@ -1204,16 +1202,6 @@ pub fn for_all_expns_in<E, F: FnMut(u32, ExpnId, &ExpnData) -> Result<(), E>>(
     Ok(())
 }
 
-pub fn for_all_data<E, F: FnMut((u32, SyntaxContext, &SyntaxContextData)) -> Result<(), E>>(
-    mut f: F,
-) -> Result<(), E> {
-    let all_data = HygieneData::with(|data| data.syntax_context_data.clone());
-    for (i, data) in all_data.into_iter().enumerate() {
-        f((i as u32, SyntaxContext(i as u32), &data))?;
-    }
-    Ok(())
-}
-
 impl<E: Encoder> Encodable<E> for ExpnId {
     default fn encode(&self, _: &mut E) -> Result<(), E::Error> {
         panic!("cannot encode `ExpnId` with `{}`", std::any::type_name::<E>());
@@ -1226,14 +1214,6 @@ impl<D: Decoder> Decodable<D> for ExpnId {
     }
 }
 
-pub fn for_all_expn_data<E, F: FnMut(u32, &ExpnData) -> Result<(), E>>(mut f: F) -> Result<(), E> {
-    let all_data = HygieneData::with(|data| data.expn_data.clone());
-    for (i, data) in all_data.into_iter().enumerate() {
-        f(i as u32, &data.unwrap_or_else(|| panic!("Missing ExpnData!")))?;
-    }
-    Ok(())
-}
-
 pub fn raw_encode_syntax_context<E: Encoder>(
     ctxt: SyntaxContext,
     context: &HygieneEncodeContext,
@@ -1362,12 +1342,6 @@ fn update_disambiguator(expn_id: ExpnId) {
         fn hash_spans(&self) -> bool {
             true
         }
-        fn byte_pos_to_line_and_col(
-            &mut self,
-            byte: BytePos,
-        ) -> Option<(Lrc<SourceFile>, usize, BytePos)> {
-            self.caching_source_map.byte_pos_to_line_and_col(byte)
-        }
         fn span_data_to_lines_and_cols(
             &mut self,
             span: &crate::SpanData,
@@ -1416,9 +1390,11 @@ fn update_disambiguator(expn_id: ExpnId) {
             let new_hash: Fingerprint = hasher.finish();
 
             HygieneData::with(|data| {
-                data.expn_data_disambiguators
-                    .get(&new_hash)
-                    .expect_none("Hash collision after disambiguator update!");
+                assert_eq!(
+                    data.expn_data_disambiguators.get(&new_hash),
+                    None,
+                    "Hash collision after disambiguator update!",
+                );
             });
         };
     }
diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs
index 4dce029e86b..6f6ff37c525 100644
--- a/compiler/rustc_span/src/lib.rs
+++ b/compiler/rustc_span/src/lib.rs
@@ -21,7 +21,6 @@
 #![feature(negative_impls)]
 #![feature(nll)]
 #![feature(min_specialization)]
-#![feature(option_expect_none)]
 
 #[macro_use]
 extern crate rustc_macros;
@@ -47,6 +46,8 @@ pub mod lev_distance;
 mod span_encoding;
 pub use span_encoding::{Span, DUMMY_SP};
 
+pub mod crate_disambiguator;
+
 pub mod symbol;
 pub use symbol::{sym, Symbol};
 
@@ -1036,10 +1037,6 @@ pub enum ExternalSourceKind {
 }
 
 impl ExternalSource {
-    pub fn is_absent(&self) -> bool {
-        !matches!(self, ExternalSource::Foreign { kind: ExternalSourceKind::Present(_), .. })
-    }
-
     pub fn get_source(&self) -> Option<&Lrc<String>> {
         match self {
             ExternalSource::Foreign { kind: ExternalSourceKind::Present(ref src), .. } => Some(src),
@@ -1432,9 +1429,6 @@ impl SourceFile {
         self.src.is_none()
     }
 
-    pub fn byte_length(&self) -> u32 {
-        self.end_pos.0 - self.start_pos.0
-    }
     pub fn count_lines(&self) -> usize {
         self.lines.len()
     }
@@ -1872,10 +1866,6 @@ pub trait HashStableContext {
     fn expn_id_cache() -> &'static LocalKey<ExpnIdCache>;
     fn hash_crate_num(&mut self, _: CrateNum, hasher: &mut StableHasher);
     fn hash_spans(&self) -> bool;
-    fn byte_pos_to_line_and_col(
-        &mut self,
-        byte: BytePos,
-    ) -> Option<(Lrc<SourceFile>, usize, BytePos)>;
     fn span_data_to_lines_and_cols(
         &mut self,
         span: &SpanData,
@@ -1904,9 +1894,10 @@ where
             return;
         }
 
+        self.ctxt().hash_stable(ctx, hasher);
+
         if self.is_dummy() {
             Hash::hash(&TAG_INVALID_SPAN, hasher);
-            self.ctxt().hash_stable(ctx, hasher);
             return;
         }
 
@@ -1919,7 +1910,6 @@ where
             Some(pos) => pos,
             None => {
                 Hash::hash(&TAG_INVALID_SPAN, hasher);
-                span.ctxt.hash_stable(ctx, hasher);
                 return;
             }
         };
@@ -1946,7 +1936,6 @@ where
         let len = (span.hi - span.lo).0;
         Hash::hash(&col_line, hasher);
         Hash::hash(&len, hasher);
-        span.ctxt.hash_stable(ctx, hasher);
     }
 }
 
@@ -1999,7 +1988,8 @@ impl<CTX: HashStableContext> HashStable<CTX> for ExpnId {
                 if cache.len() < new_len {
                     cache.resize(new_len, None);
                 }
-                cache[index].replace(sub_hash).expect_none("Cache slot was filled");
+                let prev = cache[index].replace(sub_hash);
+                assert_eq!(prev, None, "Cache slot was filled");
             });
             sub_hash.hash_stable(ctx, hasher);
         }
diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs
index b7eb6d5b379..f612d1425b9 100644
--- a/compiler/rustc_span/src/source_map.rs
+++ b/compiler/rustc_span/src/source_map.rs
@@ -453,41 +453,6 @@ impl SourceMap {
         }
     }
 
-    /// Returns `Some(span)`, a union of the LHS and RHS span. The LHS must precede the RHS. If
-    /// there are gaps between LHS and RHS, the resulting union will cross these gaps.
-    /// For this to work,
-    ///
-    ///    * the syntax contexts of both spans much match,
-    ///    * the LHS span needs to end on the same line the RHS span begins,
-    ///    * the LHS span must start at or before the RHS span.
-    pub fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option<Span> {
-        // Ensure we're at the same expansion ID.
-        if sp_lhs.ctxt() != sp_rhs.ctxt() {
-            return None;
-        }
-
-        let lhs_end = match self.lookup_line(sp_lhs.hi()) {
-            Ok(x) => x,
-            Err(_) => return None,
-        };
-        let rhs_begin = match self.lookup_line(sp_rhs.lo()) {
-            Ok(x) => x,
-            Err(_) => return None,
-        };
-
-        // If we must cross lines to merge, don't merge.
-        if lhs_end.line != rhs_begin.line {
-            return None;
-        }
-
-        // Ensure these follow the expected order and that we don't overlap.
-        if (sp_lhs.lo() <= sp_rhs.lo()) && (sp_lhs.hi() <= sp_rhs.lo()) {
-            Some(sp_lhs.to(sp_rhs))
-        } else {
-            None
-        }
-    }
-
     pub fn span_to_string(&self, sp: Span) -> String {
         if self.files.borrow().source_files.is_empty() && sp.is_dummy() {
             return "no-location".to_string();
@@ -931,13 +896,6 @@ impl SourceMap {
         SourceFileAndBytePos { sf, pos: offset }
     }
 
-    /// Converts an absolute `BytePos` to a `CharPos` relative to the `SourceFile`.
-    pub fn bytepos_to_file_charpos(&self, bpos: BytePos) -> CharPos {
-        let idx = self.lookup_source_file_idx(bpos);
-        let sf = &(*self.files.borrow().source_files)[idx];
-        sf.bytepos_to_file_charpos(bpos)
-    }
-
     // Returns the index of the `SourceFile` (in `self.files`) that contains `pos`.
     // This index is guaranteed to be valid for the lifetime of this `SourceMap`,
     // since `source_files` is a `MonotonicVec`
diff --git a/compiler/rustc_span/src/source_map/tests.rs b/compiler/rustc_span/src/source_map/tests.rs
index 0aca677248b..7d814f1d82c 100644
--- a/compiler/rustc_span/src/source_map/tests.rs
+++ b/compiler/rustc_span/src/source_map/tests.rs
@@ -10,6 +10,50 @@ fn init_source_map() -> SourceMap {
     sm
 }
 
+impl SourceMap {
+    /// Returns `Some(span)`, a union of the LHS and RHS span. The LHS must precede the RHS. If
+    /// there are gaps between LHS and RHS, the resulting union will cross these gaps.
+    /// For this to work,
+    ///
+    ///    * the syntax contexts of both spans much match,
+    ///    * the LHS span needs to end on the same line the RHS span begins,
+    ///    * the LHS span must start at or before the RHS span.
+    fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option<Span> {
+        // Ensure we're at the same expansion ID.
+        if sp_lhs.ctxt() != sp_rhs.ctxt() {
+            return None;
+        }
+
+        let lhs_end = match self.lookup_line(sp_lhs.hi()) {
+            Ok(x) => x,
+            Err(_) => return None,
+        };
+        let rhs_begin = match self.lookup_line(sp_rhs.lo()) {
+            Ok(x) => x,
+            Err(_) => return None,
+        };
+
+        // If we must cross lines to merge, don't merge.
+        if lhs_end.line != rhs_begin.line {
+            return None;
+        }
+
+        // Ensure these follow the expected order and that we don't overlap.
+        if (sp_lhs.lo() <= sp_rhs.lo()) && (sp_lhs.hi() <= sp_rhs.lo()) {
+            Some(sp_lhs.to(sp_rhs))
+        } else {
+            None
+        }
+    }
+
+    /// Converts an absolute `BytePos` to a `CharPos` relative to the `SourceFile`.
+    fn bytepos_to_file_charpos(&self, bpos: BytePos) -> CharPos {
+        let idx = self.lookup_source_file_idx(bpos);
+        let sf = &(*self.files.borrow().source_files)[idx];
+        sf.bytepos_to_file_charpos(bpos)
+    }
+}
+
 /// Tests `lookup_byte_offset`.
 #[test]
 fn t3() {
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index f9af0886a95..1d1471fdeca 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -18,7 +18,7 @@ use crate::{Edition, Span, DUMMY_SP, SESSION_GLOBALS};
 #[cfg(test)]
 mod tests;
 
-// The proc macro code for this is in `src/librustc_macros/src/symbols.rs`.
+// The proc macro code for this is in `compiler/rustc_macros/src/symbols.rs`.
 symbols! {
     // After modifying this list adjust `is_special`, `is_used_keyword`/`is_unused_keyword`,
     // this should be rarely necessary though if the keywords are kept in alphabetic order.
@@ -129,6 +129,7 @@ symbols! {
         BTreeMap,
         BTreeSet,
         BinaryHeap,
+        Borrow,
         C,
         CString,
         Center,
@@ -141,6 +142,7 @@ symbols! {
         Decodable,
         Decoder,
         Default,
+        Deref,
         Encodable,
         Encoder,
         Eq,
@@ -328,6 +330,7 @@ symbols! {
         bridge,
         bswap,
         c_str,
+        c_unwind,
         c_variadic,
         call,
         call_mut,
@@ -342,6 +345,7 @@ symbols! {
         cfg_attr,
         cfg_attr_multi,
         cfg_doctest,
+        cfg_eval,
         cfg_panic,
         cfg_sanitize,
         cfg_target_feature,
@@ -380,6 +384,7 @@ symbols! {
         const_fn_fn_ptr_basics,
         const_fn_transmute,
         const_fn_union,
+        const_generic_defaults,
         const_generics,
         const_generics_defaults,
         const_if_match,
@@ -472,6 +477,7 @@ symbols! {
         doc_cfg,
         doc_keyword,
         doc_masked,
+        doc_notable_trait,
         doc_spotlight,
         doctest,
         document_private_items,
@@ -555,6 +561,7 @@ symbols! {
         fmt,
         fmt_internals,
         fmul_fast,
+        fn_align,
         fn_must_use,
         fn_mut,
         fn_once,
@@ -732,6 +739,7 @@ symbols! {
         min_const_generics,
         min_const_unsafe_fn,
         min_specialization,
+        min_type_alias_impl_trait,
         minnumf32,
         minnumf64,
         mips_target_feature,
@@ -788,10 +796,13 @@ symbols! {
         non_modrs_mods,
         none_error,
         nontemporal_store,
-        nontrapping_dash_fptoint: "nontrapping-fptoint",
+        noop_method_borrow,
+        noop_method_clone,
+        noop_method_deref,
         noreturn,
         nostack,
         not,
+        notable_trait,
         note,
         object_safe_for_dispatch,
         of,
@@ -837,7 +848,7 @@ symbols! {
         partial_ord,
         passes,
         pat,
-        pat2018,
+        pat2015,
         pat2021,
         path,
         pattern_parentheses,
@@ -888,6 +899,8 @@ symbols! {
         profiler_runtime,
         ptr_guaranteed_eq,
         ptr_guaranteed_ne,
+        ptr_null,
+        ptr_null_mut,
         ptr_offset_from,
         pub_macro_rules,
         pub_restricted,
@@ -952,8 +965,11 @@ symbols! {
         rt,
         rtm_target_feature,
         rust,
+        rust_2015,
         rust_2015_preview,
+        rust_2018,
         rust_2018_preview,
+        rust_2021,
         rust_2021_preview,
         rust_begin_unwind,
         rust_eh_catch_typeinfo,
@@ -991,6 +1007,7 @@ symbols! {
         rustc_layout_scalar_valid_range_start,
         rustc_legacy_const_generics,
         rustc_macro_transparency,
+        rustc_main,
         rustc_mir,
         rustc_nonnull_optimization_guaranteed,
         rustc_object_lifetime_default,
@@ -1070,6 +1087,7 @@ symbols! {
         simd_lt,
         simd_mul,
         simd_ne,
+        simd_neg,
         simd_or,
         simd_reduce_add_ordered,
         simd_reduce_add_unordered,
@@ -1085,6 +1103,7 @@ symbols! {
         simd_reduce_or,
         simd_reduce_xor,
         simd_rem,
+        simd_round,
         simd_saturating_add,
         simd_saturating_sub,
         simd_scatter,
@@ -1093,6 +1112,7 @@ symbols! {
         simd_shl,
         simd_shr,
         simd_sub,
+        simd_trunc,
         simd_xor,
         since,
         sinf32,
@@ -1101,6 +1121,7 @@ symbols! {
         size_of,
         size_of_val,
         sized,
+        skip,
         slice,
         slice_alloc,
         slice_patterns,
@@ -1278,6 +1299,7 @@ symbols! {
         vreg,
         vreg_low16,
         warn,
+        wasm_abi,
         wasm_import_module,
         wasm_target_feature,
         while_let,
diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs
index b0d5f340902..7d186c330ba 100644
--- a/compiler/rustc_symbol_mangling/src/legacy.rs
+++ b/compiler/rustc_symbol_mangling/src/legacy.rs
@@ -230,7 +230,7 @@ impl Printer<'tcx> for SymbolPrinter<'tcx> {
 
     fn print_dyn_existential(
         mut self,
-        predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     ) -> Result<Self::DynExistential, Self::Error> {
         let mut first = true;
         for p in predicates {
diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs
index 7f8cded0ac0..c050bbc9b9d 100644
--- a/compiler/rustc_symbol_mangling/src/lib.rs
+++ b/compiler/rustc_symbol_mangling/src/lib.rs
@@ -90,7 +90,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(never_type)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(in_band_lifetimes)]
 #![recursion_limit = "256"]
 
@@ -198,7 +198,7 @@ fn compute_symbol_name(
     //
     // [1]: https://bugs.llvm.org/show_bug.cgi?id=44316
     if is_foreign
-        && (tcx.sess.target.arch != "wasm32"
+        && (!tcx.sess.target.is_like_wasm
             || !tcx.wasm_import_module_map(def_id.krate).contains_key(&def_id))
     {
         if let Some(name) = attrs.link_name {
diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs
index bbf7ecc39cf..37a834043f6 100644
--- a/compiler/rustc_symbol_mangling/src/v0.rs
+++ b/compiler/rustc_symbol_mangling/src/v0.rs
@@ -181,7 +181,7 @@ impl SymbolMangler<'tcx> {
 
     fn in_binder<T>(
         mut self,
-        value: &ty::Binder<T>,
+        value: &ty::Binder<'tcx, T>,
         print_value: impl FnOnce(Self, &T) -> Result<Self, !>,
     ) -> Result<Self, !>
     where
@@ -318,7 +318,7 @@ impl Printer<'tcx> for SymbolMangler<'tcx> {
 
             // Late-bound lifetimes use indices starting at 1,
             // see `BinderLevel` for more details.
-            ty::ReLateBound(debruijn, ty::BoundRegion { kind: ty::BrAnon(i) }) => {
+            ty::ReLateBound(debruijn, ty::BoundRegion { kind: ty::BrAnon(i), .. }) => {
                 let binder = &self.binders[self.binders.len() - 1 - debruijn.index()];
                 let depth = binder.lifetime_depths.start + i;
 
@@ -440,7 +440,7 @@ impl Printer<'tcx> for SymbolMangler<'tcx> {
                     }
                     match sig.abi {
                         Abi::Rust => {}
-                        Abi::C => cx.push("KC"),
+                        Abi::C { unwind: false } => cx.push("KC"),
                         abi => {
                             cx.push("K");
                             let name = abi.name();
@@ -483,7 +483,7 @@ impl Printer<'tcx> for SymbolMangler<'tcx> {
 
     fn print_dyn_existential(
         mut self,
-        predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     ) -> Result<Self::DynExistential, Self::Error> {
         for predicate in predicates {
             self = self.in_binder(&predicate, |mut cx, predicate| {
diff --git a/compiler/rustc_target/README.md b/compiler/rustc_target/README.md
index ac1e03385d1..ca72a89da5a 100644
--- a/compiler/rustc_target/README.md
+++ b/compiler/rustc_target/README.md
@@ -1,4 +1,4 @@
-`librustc_target` contains some very low-level details that are
+`rustc_target` contains some very low-level details that are
 specific to different compilation targets and so forth.
 
 For more information about how rustc works, see the [rustc dev guide].
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index 0deb1186b0f..0cf2441d84e 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -18,8 +18,7 @@ mod riscv;
 mod s390x;
 mod sparc;
 mod sparc64;
-mod wasm32;
-mod wasm32_bindgen_compat;
+mod wasm;
 mod x86;
 mod x86_64;
 mod x86_win64;
@@ -65,14 +64,17 @@ mod attr_impl {
             const NoCapture = 1 << 2;
             const NonNull   = 1 << 3;
             const ReadOnly  = 1 << 4;
-            const InReg     = 1 << 8;
+            const InReg     = 1 << 5;
+            // NoAlias on &mut arguments can only be used with LLVM >= 12 due to miscompiles
+            // in earlier versions. FIXME: Remove this distinction once possible.
+            const NoAliasMutRef = 1 << 6;
         }
     }
 }
 
 /// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum
-/// defines if this extension should be zero-extension or sign-extension when necssary. When it is
-/// not necesary to extend the argument, this enum is ignored.
+/// defines if this extension should be zero-extension or sign-extension when necessary. When it is
+/// not necessary to extend the argument, this enum is ignored.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum ArgExtension {
     None,
@@ -644,11 +646,14 @@ impl<'a, Ty> FnAbi<'a, Ty> {
             "nvptx64" => nvptx64::compute_abi_info(self),
             "hexagon" => hexagon::compute_abi_info(self),
             "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self),
-            "wasm32" => match cx.target_spec().os.as_str() {
-                "emscripten" | "wasi" => wasm32::compute_abi_info(cx, self),
-                _ => wasm32_bindgen_compat::compute_abi_info(self),
-            },
-            "asmjs" => wasm32::compute_abi_info(cx, self),
+            "wasm32" | "wasm64" => {
+                if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::Wasm {
+                    wasm::compute_wasm_abi_info(self)
+                } else {
+                    wasm::compute_c_abi_info(cx, self)
+                }
+            }
+            "asmjs" => wasm::compute_c_abi_info(cx, self),
             a => return Err(format!("unrecognized arch \"{}\" in target specification", a)),
         }
 
diff --git a/compiler/rustc_target/src/abi/call/wasm32.rs b/compiler/rustc_target/src/abi/call/wasm.rs
index ff2c0e9bb6f..bf2c08bb166 100644
--- a/compiler/rustc_target/src/abi/call/wasm32.rs
+++ b/compiler/rustc_target/src/abi/call/wasm.rs
@@ -40,7 +40,8 @@ where
     }
 }
 
-pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
+/// The purpose of this ABI is to match the C ABI (aka clang) exactly.
+pub fn compute_c_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
 where
     Ty: TyAndLayoutMethods<'a, C> + Copy,
     C: LayoutOf<Ty = Ty, TyAndLayout = TyAndLayout<'a, Ty>> + HasDataLayout,
@@ -56,3 +57,27 @@ where
         classify_arg(cx, arg);
     }
 }
+
+/// The purpose of this ABI is for matching the WebAssembly standard. This
+/// intentionally diverges from the C ABI and is specifically crafted to take
+/// advantage of LLVM's support of multiple returns in WebAssembly.
+pub fn compute_wasm_abi_info<Ty>(fn_abi: &mut FnAbi<'_, Ty>) {
+    if !fn_abi.ret.is_ignore() {
+        classify_ret(&mut fn_abi.ret);
+    }
+
+    for arg in &mut fn_abi.args {
+        if arg.is_ignore() {
+            continue;
+        }
+        classify_arg(arg);
+    }
+
+    fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
+        ret.extend_integer_width_to(32);
+    }
+
+    fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
+        arg.extend_integer_width_to(32);
+    }
+}
diff --git a/compiler/rustc_target/src/abi/call/wasm32_bindgen_compat.rs b/compiler/rustc_target/src/abi/call/wasm32_bindgen_compat.rs
deleted file mode 100644
index 59571fd9d48..00000000000
--- a/compiler/rustc_target/src/abi/call/wasm32_bindgen_compat.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-// This is not and has never been a correct C ABI for WebAssembly, but
-// for a long time this was the C ABI that Rust used. wasm-bindgen
-// depends on ABI details for this ABI and is incompatible with the
-// correct C ABI, so this ABI is being kept around until wasm-bindgen
-// can be fixed to work with the correct ABI. See #63649 for further
-// discussion.
-
-use crate::abi::call::{ArgAbi, FnAbi};
-
-fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
-    ret.extend_integer_width_to(32);
-}
-
-fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
-    arg.extend_integer_width_to(32);
-}
-
-pub fn compute_abi_info<Ty>(fn_abi: &mut FnAbi<'_, Ty>) {
-    if !fn_abi.ret.is_ignore() {
-        classify_ret(&mut fn_abi.ret);
-    }
-
-    for arg in &mut fn_abi.args {
-        if arg.is_ignore() {
-            continue;
-        }
-        classify_arg(arg);
-    }
-}
diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs
index b14b1ef00db..e2618da749f 100644
--- a/compiler/rustc_target/src/abi/mod.rs
+++ b/compiler/rustc_target/src/abi/mod.rs
@@ -1112,7 +1112,7 @@ pub enum PointerKind {
     /// `&T` where `T` contains no `UnsafeCell`, is `noalias` and `readonly`.
     Frozen,
 
-    /// `&mut T`, when we know `noalias` is safe for LLVM.
+    /// `&mut T` which is `noalias` but not `readonly`.
     UniqueBorrowed,
 
     /// `Box<T>`, unlike `UniqueBorrowed`, it also has `noalias` on returns.
diff --git a/compiler/rustc_target/src/asm/arm.rs b/compiler/rustc_target/src/asm/arm.rs
index 28000916e0c..a7a708fe7de 100644
--- a/compiler/rustc_target/src/asm/arm.rs
+++ b/compiler/rustc_target/src/asm/arm.rs
@@ -68,7 +68,6 @@ fn frame_pointer_r11(
     _arch: InlineAsmArch,
     has_feature: impl FnMut(&str) -> bool,
     target: &Target,
-    _allocating: bool,
 ) -> Result<(), &'static str> {
     if !frame_pointer_is_r7(has_feature, target) {
         Err("the frame pointer (r11) cannot be used as an operand for inline asm")
@@ -81,7 +80,6 @@ fn frame_pointer_r7(
     _arch: InlineAsmArch,
     has_feature: impl FnMut(&str) -> bool,
     target: &Target,
-    _allocating: bool,
 ) -> Result<(), &'static str> {
     if frame_pointer_is_r7(has_feature, target) {
         Err("the frame pointer (r7) cannot be used as an operand for inline asm")
diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs
index dfc0989a9f8..e2268a61a42 100644
--- a/compiler/rustc_target/src/asm/mod.rs
+++ b/compiler/rustc_target/src/asm/mod.rs
@@ -90,7 +90,7 @@ macro_rules! def_regs {
                 match name {
                     $(
                         $($alias)|* | $reg_name => {
-                            $($filter(_arch, &mut _has_feature, _target, false)?;)?
+                            $($filter(_arch, &mut _has_feature, _target)?;)?
                             Ok(Self::$reg)
                         }
                     )*
@@ -114,7 +114,7 @@ macro_rules! def_regs {
             #[allow(unused_imports)]
             use super::{InlineAsmReg, InlineAsmRegClass};
             $(
-                if $($filter(_arch, &mut _has_feature, _target, true).is_ok() &&)? true {
+                if $($filter(_arch, &mut _has_feature, _target).is_ok() &&)? true {
                     if let Some(set) = _map.get_mut(&InlineAsmRegClass::$arch($arch_regclass::$class)) {
                         set.insert(InlineAsmReg::$arch($arch_reg::$reg));
                     }
@@ -229,6 +229,8 @@ pub enum InlineAsmReg {
     Mips(MipsInlineAsmReg),
     SpirV(SpirVInlineAsmReg),
     Wasm(WasmInlineAsmReg),
+    // Placeholder for invalid register constraints for the current target
+    Err,
 }
 
 impl InlineAsmReg {
@@ -240,6 +242,7 @@ impl InlineAsmReg {
             Self::RiscV(r) => r.name(),
             Self::Hexagon(r) => r.name(),
             Self::Mips(r) => r.name(),
+            Self::Err => "<reg>",
         }
     }
 
@@ -251,6 +254,7 @@ impl InlineAsmReg {
             Self::RiscV(r) => InlineAsmRegClass::RiscV(r.reg_class()),
             Self::Hexagon(r) => InlineAsmRegClass::Hexagon(r.reg_class()),
             Self::Mips(r) => InlineAsmRegClass::Mips(r.reg_class()),
+            Self::Err => InlineAsmRegClass::Err,
         }
     }
 
@@ -309,6 +313,7 @@ impl InlineAsmReg {
             Self::RiscV(r) => r.emit(out, arch, modifier),
             Self::Hexagon(r) => r.emit(out, arch, modifier),
             Self::Mips(r) => r.emit(out, arch, modifier),
+            Self::Err => unreachable!("Use of InlineAsmReg::Err"),
         }
     }
 
@@ -320,6 +325,7 @@ impl InlineAsmReg {
             Self::RiscV(_) => cb(self),
             Self::Hexagon(r) => r.overlapping_regs(|r| cb(Self::Hexagon(r))),
             Self::Mips(_) => cb(self),
+            Self::Err => unreachable!("Use of InlineAsmReg::Err"),
         }
     }
 }
@@ -346,6 +352,8 @@ pub enum InlineAsmRegClass {
     Mips(MipsInlineAsmRegClass),
     SpirV(SpirVInlineAsmRegClass),
     Wasm(WasmInlineAsmRegClass),
+    // Placeholder for invalid register constraints for the current target
+    Err,
 }
 
 impl InlineAsmRegClass {
@@ -360,6 +368,7 @@ impl InlineAsmRegClass {
             Self::Mips(r) => r.name(),
             Self::SpirV(r) => r.name(),
             Self::Wasm(r) => r.name(),
+            Self::Err => rustc_span::symbol::sym::reg,
         }
     }
 
@@ -377,6 +386,7 @@ impl InlineAsmRegClass {
             Self::Mips(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Mips),
             Self::SpirV(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::SpirV),
             Self::Wasm(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Wasm),
+            Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
 
@@ -401,6 +411,7 @@ impl InlineAsmRegClass {
             Self::Mips(r) => r.suggest_modifier(arch, ty),
             Self::SpirV(r) => r.suggest_modifier(arch, ty),
             Self::Wasm(r) => r.suggest_modifier(arch, ty),
+            Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
 
@@ -421,6 +432,7 @@ impl InlineAsmRegClass {
             Self::Mips(r) => r.default_modifier(arch),
             Self::SpirV(r) => r.default_modifier(arch),
             Self::Wasm(r) => r.default_modifier(arch),
+            Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
 
@@ -440,6 +452,7 @@ impl InlineAsmRegClass {
             Self::Mips(r) => r.supported_types(arch),
             Self::SpirV(r) => r.supported_types(arch),
             Self::Wasm(r) => r.supported_types(arch),
+            Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
 
@@ -476,6 +489,7 @@ impl InlineAsmRegClass {
             Self::Mips(r) => r.valid_modifiers(arch),
             Self::SpirV(r) => r.valid_modifiers(arch),
             Self::Wasm(r) => r.valid_modifiers(arch),
+            Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
 }
diff --git a/compiler/rustc_target/src/asm/riscv.rs b/compiler/rustc_target/src/asm/riscv.rs
index ced7483b005..185d6ac8246 100644
--- a/compiler/rustc_target/src/asm/riscv.rs
+++ b/compiler/rustc_target/src/asm/riscv.rs
@@ -52,7 +52,6 @@ fn not_e(
     _arch: InlineAsmArch,
     mut has_feature: impl FnMut(&str) -> bool,
     _target: &Target,
-    _allocating: bool,
 ) -> Result<(), &'static str> {
     if has_feature("e") {
         Err("register can't be used with the `e` target feature")
diff --git a/compiler/rustc_target/src/asm/x86.rs b/compiler/rustc_target/src/asm/x86.rs
index 0f62c19e1a3..90660dad4c2 100644
--- a/compiler/rustc_target/src/asm/x86.rs
+++ b/compiler/rustc_target/src/asm/x86.rs
@@ -133,7 +133,6 @@ fn x86_64_only(
     arch: InlineAsmArch,
     _has_feature: impl FnMut(&str) -> bool,
     _target: &Target,
-    _allocating: bool,
 ) -> Result<(), &'static str> {
     match arch {
         InlineAsmArch::X86 => Err("register is only available on x86_64"),
@@ -146,13 +145,9 @@ fn high_byte(
     arch: InlineAsmArch,
     _has_feature: impl FnMut(&str) -> bool,
     _target: &Target,
-    allocating: bool,
 ) -> Result<(), &'static str> {
     match arch {
-        InlineAsmArch::X86_64 if allocating => {
-            // The error message isn't actually used...
-            Err("high byte registers are not allocated by reg_byte")
-        }
+        InlineAsmArch::X86_64 => Err("high byte registers cannot be used as an operand on x86_64"),
         _ => Ok(()),
     }
 }
diff --git a/compiler/rustc_target/src/lib.rs b/compiler/rustc_target/src/lib.rs
index fb747dfcbd3..b54764b9e32 100644
--- a/compiler/rustc_target/src/lib.rs
+++ b/compiler/rustc_target/src/lib.rs
@@ -28,5 +28,5 @@ pub mod spec;
 
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
-/// instead of implementing everything in librustc_middle.
+/// instead of implementing everything in `rustc_middle`.
 pub trait HashStableContext {}
diff --git a/compiler/rustc_target/src/spec/aarch64_apple_darwin.rs b/compiler/rustc_target/src/spec/aarch64_apple_darwin.rs
index 7de809f7622..feadd4e891c 100644
--- a/compiler/rustc_target/src/spec/aarch64_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/aarch64_apple_darwin.rs
@@ -1,11 +1,12 @@
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::apple_base::opts("macos");
     base.cpu = "apple-a12".to_string();
     base.max_atomic_width = Some(128);
-    base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-arch".to_string(), "arm64".to_string()]);
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::THREAD;
 
+    base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-arch".to_string(), "arm64".to_string()]);
     base.link_env_remove.extend(super::apple_base::macos_link_env_remove());
 
     // Clang automatically chooses a more specific target based on
diff --git a/compiler/rustc_target/src/spec/aarch64_fuchsia.rs b/compiler/rustc_target/src/spec/aarch64_fuchsia.rs
index 1252741f979..c9cb21f1eb1 100644
--- a/compiler/rustc_target/src/spec/aarch64_fuchsia.rs
+++ b/compiler/rustc_target/src/spec/aarch64_fuchsia.rs
@@ -1,8 +1,9 @@
-use crate::spec::{Target, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::fuchsia_base::opts();
     base.max_atomic_width = Some(128);
+    base.supported_sanitizers = SanitizerSet::ADDRESS;
 
     Target {
         llvm_target: "aarch64-fuchsia".to_string(),
diff --git a/compiler/rustc_target/src/spec/aarch64_linux_android.rs b/compiler/rustc_target/src/spec/aarch64_linux_android.rs
index fa6108df206..eaf3a2dbcf8 100644
--- a/compiler/rustc_target/src/spec/aarch64_linux_android.rs
+++ b/compiler/rustc_target/src/spec/aarch64_linux_android.rs
@@ -1,4 +1,4 @@
-use crate::spec::{Target, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetOptions};
 
 // See https://developer.android.com/ndk/guides/abis.html#arm64-v8a
 // for target ABI requirements.
@@ -9,6 +9,7 @@ pub fn target() -> Target {
     // As documented in http://developer.android.com/ndk/guides/cpu-features.html
     // the neon (ASIMD) and FP must exist on all android aarch64 targets.
     base.features = "+neon,+fp-armv8".to_string();
+    base.supported_sanitizers = SanitizerSet::HWADDRESS;
     Target {
         llvm_target: "aarch64-linux-android".to_string(),
         pointer_width: 64,
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs
index 58c72af4e76..a07cd7db889 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs
@@ -1,8 +1,13 @@
-use crate::spec::{Target, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.max_atomic_width = Some(128);
+    base.supported_sanitizers = SanitizerSet::ADDRESS
+        | SanitizerSet::LEAK
+        | SanitizerSet::MEMORY
+        | SanitizerSet::THREAD
+        | SanitizerSet::HWADDRESS;
 
     Target {
         llvm_target: "aarch64-unknown-linux-gnu".to_string(),
diff --git a/compiler/rustc_target/src/spec/abi.rs b/compiler/rustc_target/src/spec/abi.rs
index 65e8a4e8db2..a026a623f78 100644
--- a/compiler/rustc_target/src/spec/abi.rs
+++ b/compiler/rustc_target/src/spec/abi.rs
@@ -8,24 +8,21 @@ mod tests;
 #[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy, Debug)]
 #[derive(HashStable_Generic, Encodable, Decodable)]
 pub enum Abi {
-    // N.B., this ordering MUST match the AbiDatas array below.
-    // (This is ensured by the test indices_are_correct().)
-
     // Multiplatform / generic ABIs
     //
     // These ABIs come first because every time we add a new ABI, we
     // have to re-bless all the hashing tests. These are used in many
     // places, so giving them stable values reduces test churn. The
     // specific values are meaningless.
-    Rust = 0,
-    C = 1,
+    Rust,
+    C { unwind: bool },
 
     // Single platform ABIs
     Cdecl,
-    Stdcall,
+    Stdcall { unwind: bool },
     Fastcall,
     Vectorcall,
-    Thiscall,
+    Thiscall { unwind: bool },
     Aapcs,
     Win64,
     SysV64,
@@ -37,9 +34,10 @@ pub enum Abi {
     AvrInterrupt,
     AvrNonBlockingInterrupt,
     CCmseNonSecureCall,
+    Wasm,
 
     // Multiplatform / generic ABIs
-    System,
+    System { unwind: bool },
     RustIntrinsic,
     RustCall,
     PlatformIntrinsic,
@@ -61,13 +59,16 @@ pub struct AbiData {
 const AbiDatas: &[AbiData] = &[
     // Cross-platform ABIs
     AbiData { abi: Abi::Rust, name: "Rust", generic: true },
-    AbiData { abi: Abi::C, name: "C", generic: true },
+    AbiData { abi: Abi::C { unwind: false }, name: "C", generic: true },
+    AbiData { abi: Abi::C { unwind: true }, name: "C-unwind", generic: true },
     // Platform-specific ABIs
     AbiData { abi: Abi::Cdecl, name: "cdecl", generic: false },
-    AbiData { abi: Abi::Stdcall, name: "stdcall", generic: false },
+    AbiData { abi: Abi::Stdcall { unwind: false }, name: "stdcall", generic: false },
+    AbiData { abi: Abi::Stdcall { unwind: true }, name: "stdcall-unwind", generic: false },
     AbiData { abi: Abi::Fastcall, name: "fastcall", generic: false },
     AbiData { abi: Abi::Vectorcall, name: "vectorcall", generic: false },
-    AbiData { abi: Abi::Thiscall, name: "thiscall", generic: false },
+    AbiData { abi: Abi::Thiscall { unwind: false }, name: "thiscall", generic: false },
+    AbiData { abi: Abi::Thiscall { unwind: true }, name: "thiscall-unwind", generic: false },
     AbiData { abi: Abi::Aapcs, name: "aapcs", generic: false },
     AbiData { abi: Abi::Win64, name: "win64", generic: false },
     AbiData { abi: Abi::SysV64, name: "sysv64", generic: false },
@@ -83,8 +84,10 @@ const AbiDatas: &[AbiData] = &[
         generic: false,
     },
     AbiData { abi: Abi::CCmseNonSecureCall, name: "C-cmse-nonsecure-call", generic: false },
+    AbiData { abi: Abi::Wasm, name: "wasm", generic: false },
     // Cross-platform ABIs
-    AbiData { abi: Abi::System, name: "system", generic: true },
+    AbiData { abi: Abi::System { unwind: false }, name: "system", generic: true },
+    AbiData { abi: Abi::System { unwind: true }, name: "system-unwind", generic: true },
     AbiData { abi: Abi::RustIntrinsic, name: "rust-intrinsic", generic: true },
     AbiData { abi: Abi::RustCall, name: "rust-call", generic: true },
     AbiData { abi: Abi::PlatformIntrinsic, name: "platform-intrinsic", generic: true },
@@ -103,7 +106,53 @@ pub fn all_names() -> Vec<&'static str> {
 impl Abi {
     #[inline]
     pub fn index(self) -> usize {
-        self as usize
+        // N.B., this ordering MUST match the AbiDatas array above.
+        // (This is ensured by the test indices_are_correct().)
+        use Abi::*;
+        let i = match self {
+            // Cross-platform ABIs
+            Rust => 0,
+            C { unwind: false } => 1,
+            C { unwind: true } => 2,
+            // Platform-specific ABIs
+            Cdecl => 3,
+            Stdcall { unwind: false } => 4,
+            Stdcall { unwind: true } => 5,
+            Fastcall => 6,
+            Vectorcall => 7,
+            Thiscall { unwind: false } => 8,
+            Thiscall { unwind: true } => 9,
+            Aapcs => 10,
+            Win64 => 11,
+            SysV64 => 12,
+            PtxKernel => 13,
+            Msp430Interrupt => 14,
+            X86Interrupt => 15,
+            AmdGpuKernel => 16,
+            EfiApi => 17,
+            AvrInterrupt => 18,
+            AvrNonBlockingInterrupt => 19,
+            CCmseNonSecureCall => 20,
+            Wasm => 21,
+            // Cross-platform ABIs
+            System { unwind: false } => 22,
+            System { unwind: true } => 23,
+            RustIntrinsic => 24,
+            RustCall => 25,
+            PlatformIntrinsic => 26,
+            Unadjusted => 27,
+        };
+        debug_assert!(
+            AbiDatas
+                .iter()
+                .enumerate()
+                .find(|(_, AbiData { abi, .. })| *abi == self)
+                .map(|(index, _)| index)
+                .expect("abi variant has associated data")
+                == i,
+            "Abi index did not match `AbiDatas` ordering"
+        );
+        i
     }
 
     #[inline]
@@ -122,6 +171,8 @@ impl Abi {
 
 impl fmt::Display for Abi {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "\"{}\"", self.name())
+        match self {
+            abi => write!(f, "\"{}\"", abi.name()),
+        }
     }
 }
diff --git a/compiler/rustc_target/src/spec/android_base.rs b/compiler/rustc_target/src/spec/android_base.rs
index f6fbe7cd5f6..aaf81648c51 100644
--- a/compiler/rustc_target/src/spec/android_base.rs
+++ b/compiler/rustc_target/src/spec/android_base.rs
@@ -6,13 +6,16 @@ pub fn opts() -> TargetOptions {
     // Many of the symbols defined in compiler-rt are also defined in libgcc.
     // Android's linker doesn't like that by default.
     base.pre_link_args
-        .get_mut(&LinkerFlavor::Gcc)
-        .unwrap()
+        .entry(LinkerFlavor::Gcc)
+        .or_default()
         .push("-Wl,--allow-multiple-definition".to_string());
     base.dwarf_version = Some(2);
     base.position_independent_executables = true;
     base.has_elf_tls = false;
-    base.requires_uwtable = true;
+    // This is for backward compatibility, see https://github.com/rust-lang/rust/issues/49867
+    // for context. (At that time, there was no `-C force-unwind-tables`, so the only solution
+    // was to always emit `uwtable`).
+    base.default_uwtable = true;
     base.crt_static_respected = false;
     base
 }
diff --git a/compiler/rustc_target/src/spec/apple_base.rs b/compiler/rustc_target/src/spec/apple_base.rs
index 23f1357af16..6fa0b345450 100644
--- a/compiler/rustc_target/src/spec/apple_base.rs
+++ b/compiler/rustc_target/src/spec/apple_base.rs
@@ -1,6 +1,6 @@
 use std::env;
 
-use crate::spec::{LinkArgs, SplitDebuginfo, TargetOptions};
+use crate::spec::{SplitDebuginfo, TargetOptions};
 
 pub fn opts(os: &str) -> TargetOptions {
     // ELF TLS is only available in macOS 10.7+. If you try to compile for 10.6
@@ -27,10 +27,8 @@ pub fn opts(os: &str) -> TargetOptions {
         is_like_osx: true,
         dwarf_version: Some(2),
         has_rpath: true,
-        dll_prefix: "lib".to_string(),
         dll_suffix: ".dylib".to_string(),
         archive_format: "darwin".to_string(),
-        pre_link_args: LinkArgs::new(),
         has_elf_tls: version >= (10, 7),
         abi_return_struct_as_int: true,
         emit_debug_gdb_scripts: false,
diff --git a/compiler/rustc_target/src/spec/arm_base.rs b/compiler/rustc_target/src/spec/arm_base.rs
index b74d80dc6bb..01f573313c9 100644
--- a/compiler/rustc_target/src/spec/arm_base.rs
+++ b/compiler/rustc_target/src/spec/arm_base.rs
@@ -2,5 +2,14 @@ use crate::spec::abi::Abi;
 
 // All the calling conventions trigger an assertion(Unsupported calling convention) in llvm on arm
 pub fn unsupported_abis() -> Vec<Abi> {
-    vec![Abi::Stdcall, Abi::Fastcall, Abi::Vectorcall, Abi::Thiscall, Abi::Win64, Abi::SysV64]
+    vec![
+        Abi::Stdcall { unwind: false },
+        Abi::Stdcall { unwind: true },
+        Abi::Fastcall,
+        Abi::Vectorcall,
+        Abi::Thiscall { unwind: false },
+        Abi::Thiscall { unwind: true },
+        Abi::Win64,
+        Abi::SysV64,
+    ]
 }
diff --git a/compiler/rustc_target/src/spec/armv7_linux_androideabi.rs b/compiler/rustc_target/src/spec/armv7_linux_androideabi.rs
index 9aa378a8018..02a1191463e 100644
--- a/compiler/rustc_target/src/spec/armv7_linux_androideabi.rs
+++ b/compiler/rustc_target/src/spec/armv7_linux_androideabi.rs
@@ -12,7 +12,7 @@ pub fn target() -> Target {
     let mut base = super::android_base::opts();
     base.features = "+v7,+thumb-mode,+thumb2,+vfp3,-d32,-neon".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-march=armv7-a".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-march=armv7-a".to_string());
 
     Target {
         llvm_target: "armv7-none-linux-android".to_string(),
diff --git a/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabi.rs b/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabi.rs
index ae6b8286f08..f6fe88de37c 100644
--- a/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabi.rs
+++ b/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabi.rs
@@ -13,7 +13,6 @@ pub fn target() -> Target {
 
         options: TargetOptions {
             features: "+v7,+thumb2,+soft-float,-neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             mcount: "\u{1}__gnu_mcount_nc".to_string(),
diff --git a/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabihf.rs b/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabihf.rs
index 48c16b620fd..5f0f47dd397 100644
--- a/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabihf.rs
+++ b/compiler/rustc_target/src/spec/armv7_unknown_linux_gnueabihf.rs
@@ -14,7 +14,6 @@ pub fn target() -> Target {
         options: TargetOptions {
             // Info about features at https://wiki.debian.org/ArmHardFloatPort
             features: "+v7,+vfp3,-d32,+thumb2,-neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             mcount: "\u{1}__gnu_mcount_nc".to_string(),
diff --git a/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabi.rs b/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabi.rs
index 9f9f1bd79b0..c888fc2d4a3 100644
--- a/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabi.rs
+++ b/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabi.rs
@@ -18,7 +18,6 @@ pub fn target() -> Target {
 
         options: TargetOptions {
             features: "+v7,+thumb2,+soft-float,-neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             mcount: "\u{1}mcount".to_string(),
diff --git a/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabihf.rs b/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabihf.rs
index 59deee30ef2..2432ea519a8 100644
--- a/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabihf.rs
+++ b/compiler/rustc_target/src/spec/armv7_unknown_linux_musleabihf.rs
@@ -17,7 +17,6 @@ pub fn target() -> Target {
         // target.
         options: TargetOptions {
             features: "+v7,+vfp3,-d32,+thumb2,-neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             mcount: "\u{1}mcount".to_string(),
diff --git a/compiler/rustc_target/src/spec/armv7_unknown_netbsd_eabihf.rs b/compiler/rustc_target/src/spec/armv7_unknown_netbsd_eabihf.rs
index 660525704c1..4fae3a8d0bf 100644
--- a/compiler/rustc_target/src/spec/armv7_unknown_netbsd_eabihf.rs
+++ b/compiler/rustc_target/src/spec/armv7_unknown_netbsd_eabihf.rs
@@ -11,7 +11,6 @@ pub fn target() -> Target {
         options: TargetOptions {
             env: "eabihf".to_string(),
             features: "+v7,+vfp3,-d32,+thumb2,-neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             mcount: "__mcount".to_string(),
diff --git a/compiler/rustc_target/src/spec/armv7_wrs_vxworks_eabihf.rs b/compiler/rustc_target/src/spec/armv7_wrs_vxworks_eabihf.rs
index 6a43054067f..9fe7098a85f 100644
--- a/compiler/rustc_target/src/spec/armv7_wrs_vxworks_eabihf.rs
+++ b/compiler/rustc_target/src/spec/armv7_wrs_vxworks_eabihf.rs
@@ -10,7 +10,6 @@ pub fn target() -> Target {
         options: TargetOptions {
             // Info about features at https://wiki.debian.org/ArmHardFloatPort
             features: "+v7,+vfp3,-d32,+thumb2,-neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             ..base
diff --git a/compiler/rustc_target/src/spec/avr_gnu_base.rs b/compiler/rustc_target/src/spec/avr_gnu_base.rs
index 67a7684da2c..69ccce875ab 100644
--- a/compiler/rustc_target/src/spec/avr_gnu_base.rs
+++ b/compiler/rustc_target/src/spec/avr_gnu_base.rs
@@ -15,28 +15,12 @@ pub fn target(target_cpu: String) -> Target {
             exe_suffix: ".elf".to_string(),
 
             linker: Some("avr-gcc".to_owned()),
-            dynamic_linking: false,
             executables: true,
             linker_is_gnu: true,
-            has_rpath: false,
-            position_independent_executables: false,
             eh_frame_header: false,
-            pre_link_args: vec![(
-                LinkerFlavor::Gcc,
-                vec![
-                    format!("-mmcu={}", target_cpu),
-                    // We want to be able to strip as much executable code as possible
-                    // from the linker command line, and this flag indicates to the
-                    // linker that it can avoid linking in dynamic libraries that don't
-                    // actually satisfy any symbols up to that point (as with many other
-                    // resolutions the linker does). This option only applies to all
-                    // following libraries so we're sure to pass it as one of the first
-                    // arguments.
-                    "-Wl,--as-needed".to_string(),
-                ],
-            )]
-            .into_iter()
-            .collect(),
+            pre_link_args: vec![(LinkerFlavor::Gcc, vec![format!("-mmcu={}", target_cpu)])]
+                .into_iter()
+                .collect(),
             late_link_args: vec![(LinkerFlavor::Gcc, vec!["-lgcc".to_owned()])]
                 .into_iter()
                 .collect(),
diff --git a/compiler/rustc_target/src/spec/crt_objects.rs b/compiler/rustc_target/src/spec/crt_objects.rs
index 51a48147e6b..2fc9ab29f92 100644
--- a/compiler/rustc_target/src/spec/crt_objects.rs
+++ b/compiler/rustc_target/src/spec/crt_objects.rs
@@ -108,11 +108,13 @@ pub(super) fn post_mingw() -> CrtObjects {
 }
 
 pub(super) fn pre_wasi_fallback() -> CrtObjects {
+    // Use crt1-command.o instead of crt1.o to enable support for new-style
+    // commands. See https://reviews.llvm.org/D81689 for more info.
     new(&[
-        (LinkOutputKind::DynamicNoPicExe, &["crt1.o"]),
-        (LinkOutputKind::DynamicPicExe, &["crt1.o"]),
-        (LinkOutputKind::StaticNoPicExe, &["crt1.o"]),
-        (LinkOutputKind::StaticPicExe, &["crt1.o"]),
+        (LinkOutputKind::DynamicNoPicExe, &["crt1-command.o"]),
+        (LinkOutputKind::DynamicPicExe, &["crt1-command.o"]),
+        (LinkOutputKind::StaticNoPicExe, &["crt1-command.o"]),
+        (LinkOutputKind::StaticPicExe, &["crt1-command.o"]),
         (LinkOutputKind::WasiReactorExe, &["crt1-reactor.o"]),
     ])
 }
diff --git a/compiler/rustc_target/src/spec/dragonfly_base.rs b/compiler/rustc_target/src/spec/dragonfly_base.rs
index b96de7ab1ed..dd017098782 100644
--- a/compiler/rustc_target/src/spec/dragonfly_base.rs
+++ b/compiler/rustc_target/src/spec/dragonfly_base.rs
@@ -1,20 +1,6 @@
-use crate::spec::{LinkArgs, LinkerFlavor, RelroLevel, TargetOptions};
+use crate::spec::{RelroLevel, TargetOptions};
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(
-        LinkerFlavor::Gcc,
-        vec![
-            // GNU-style linkers will use this to omit linking to libraries
-            // which don't actually fulfill any relocations, but only for
-            // libraries which follow this flag.  Thus, use it before
-            // specifying libraries to link to.
-            "-Wl,--as-needed".to_string(),
-            // Always enable NX protection when it is available
-            "-Wl,-z,noexecstack".to_string(),
-        ],
-    );
-
     TargetOptions {
         os: "dragonfly".to_string(),
         dynamic_linking: true,
@@ -22,7 +8,6 @@ pub fn opts() -> TargetOptions {
         os_family: Some("unix".to_string()),
         linker_is_gnu: true,
         has_rpath: true,
-        pre_link_args: args,
         position_independent_executables: true,
         relro_level: RelroLevel::Full,
         dwarf_version: Some(2),
diff --git a/compiler/rustc_target/src/spec/freebsd_base.rs b/compiler/rustc_target/src/spec/freebsd_base.rs
index c70c492716b..ad3383cc5f2 100644
--- a/compiler/rustc_target/src/spec/freebsd_base.rs
+++ b/compiler/rustc_target/src/spec/freebsd_base.rs
@@ -1,20 +1,6 @@
-use crate::spec::{LinkArgs, LinkerFlavor, RelroLevel, TargetOptions};
+use crate::spec::{RelroLevel, TargetOptions};
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(
-        LinkerFlavor::Gcc,
-        vec![
-            // GNU-style linkers will use this to omit linking to libraries
-            // which don't actually fulfill any relocations, but only for
-            // libraries which follow this flag.  Thus, use it before
-            // specifying libraries to link to.
-            "-Wl,--as-needed".to_string(),
-            // Always enable NX protection when it is available
-            "-Wl,-z,noexecstack".to_string(),
-        ],
-    );
-
     TargetOptions {
         os: "freebsd".to_string(),
         dynamic_linking: true,
@@ -22,7 +8,6 @@ pub fn opts() -> TargetOptions {
         os_family: Some("unix".to_string()),
         linker_is_gnu: true,
         has_rpath: true,
-        pre_link_args: args,
         position_independent_executables: true,
         eliminate_frame_pointer: false, // FIXME 43575
         relro_level: RelroLevel::Full,
diff --git a/compiler/rustc_target/src/spec/fuchsia_base.rs b/compiler/rustc_target/src/spec/fuchsia_base.rs
index 5c39773cbe3..2b925f8b946 100644
--- a/compiler/rustc_target/src/spec/fuchsia_base.rs
+++ b/compiler/rustc_target/src/spec/fuchsia_base.rs
@@ -23,13 +23,11 @@ pub fn opts() -> TargetOptions {
         os: "fuchsia".to_string(),
         linker_flavor: LinkerFlavor::Lld(LldFlavor::Ld),
         linker: Some("rust-lld".to_owned()),
-        lld_flavor: LldFlavor::Ld,
         dynamic_linking: true,
         executables: true,
         os_family: Some("unix".to_string()),
         is_like_fuchsia: true,
         linker_is_gnu: true,
-        has_rpath: false,
         pre_link_args,
         pre_link_objects: crt_objects::new(&[
             (LinkOutputKind::DynamicNoPicExe, &["Scrt1.o"]),
diff --git a/compiler/rustc_target/src/spec/haiku_base.rs b/compiler/rustc_target/src/spec/haiku_base.rs
index ec87645c4fa..956e4ed4bf9 100644
--- a/compiler/rustc_target/src/spec/haiku_base.rs
+++ b/compiler/rustc_target/src/spec/haiku_base.rs
@@ -5,7 +5,6 @@ pub fn opts() -> TargetOptions {
         os: "haiku".to_string(),
         dynamic_linking: true,
         executables: true,
-        has_rpath: false,
         os_family: Some("unix".to_string()),
         relro_level: RelroLevel::Full,
         linker_is_gnu: true,
diff --git a/compiler/rustc_target/src/spec/hermit_base.rs b/compiler/rustc_target/src/spec/hermit_base.rs
index a75158a0ea0..ad013047e6a 100644
--- a/compiler/rustc_target/src/spec/hermit_base.rs
+++ b/compiler/rustc_target/src/spec/hermit_base.rs
@@ -1,5 +1,4 @@
-use crate::spec::{LinkArgs, LinkerFlavor, LldFlavor, PanicStrategy};
-use crate::spec::{RelocModel, TargetOptions, TlsModel};
+use crate::spec::{LinkArgs, LinkerFlavor, LldFlavor, PanicStrategy, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     let mut pre_link_args = LinkArgs::new();
@@ -19,8 +18,6 @@ pub fn opts() -> TargetOptions {
         panic_strategy: PanicStrategy::Abort,
         position_independent_executables: true,
         static_position_independent_executables: true,
-        relocation_model: RelocModel::Pic,
-        os_family: None,
         tls_model: TlsModel::InitialExec,
         ..Default::default()
     }
diff --git a/compiler/rustc_target/src/spec/hermit_kernel_base.rs b/compiler/rustc_target/src/spec/hermit_kernel_base.rs
index 622f0d9a471..6d18a14d6ae 100644
--- a/compiler/rustc_target/src/spec/hermit_kernel_base.rs
+++ b/compiler/rustc_target/src/spec/hermit_kernel_base.rs
@@ -1,5 +1,4 @@
-use crate::spec::{LinkArgs, LinkerFlavor, LldFlavor, PanicStrategy};
-use crate::spec::{RelocModel, TargetOptions, TlsModel};
+use crate::spec::{LinkArgs, LinkerFlavor, LldFlavor, PanicStrategy, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     let mut pre_link_args = LinkArgs::new();
@@ -20,8 +19,6 @@ pub fn opts() -> TargetOptions {
         panic_strategy: PanicStrategy::Abort,
         position_independent_executables: true,
         static_position_independent_executables: true,
-        relocation_model: RelocModel::Pic,
-        os_family: None,
         tls_model: TlsModel::InitialExec,
         ..Default::default()
     }
diff --git a/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs
index 73d5e2057f9..e0097ee220a 100644
--- a/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs
@@ -1,4 +1,4 @@
-use crate::spec::{LinkArgs, Target};
+use crate::spec::Target;
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
@@ -8,15 +8,11 @@ pub fn target() -> Target {
     base.features = "-small-data,+hvx-length128b".to_string();
 
     base.crt_static_default = false;
-    base.atomic_cas = true;
     base.has_rpath = true;
     base.linker_is_gnu = false;
     base.dynamic_linking = true;
     base.executables = true;
 
-    base.pre_link_args = LinkArgs::new();
-    base.post_link_args = LinkArgs::new();
-
     Target {
         llvm_target: "hexagon-unknown-linux-musl".to_string(),
         pointer_width: 32,
diff --git a/compiler/rustc_target/src/spec/i686_pc_windows_gnu.rs b/compiler/rustc_target/src/spec/i686_pc_windows_gnu.rs
index 4979a5b3bc8..7002d88c512 100644
--- a/compiler/rustc_target/src/spec/i686_pc_windows_gnu.rs
+++ b/compiler/rustc_target/src/spec/i686_pc_windows_gnu.rs
@@ -12,8 +12,8 @@ pub fn target() -> Target {
     // Mark all dynamic libraries and executables as compatible with the larger 4GiB address
     // space available to x86 Windows binaries on x86_64.
     base.pre_link_args
-        .get_mut(&LinkerFlavor::Gcc)
-        .unwrap()
+        .entry(LinkerFlavor::Gcc)
+        .or_default()
         .push("-Wl,--large-address-aware".to_string());
 
     Target {
diff --git a/compiler/rustc_target/src/spec/i686_pc_windows_msvc.rs b/compiler/rustc_target/src/spec/i686_pc_windows_msvc.rs
index e7a5643eaaa..74074cfb5dd 100644
--- a/compiler/rustc_target/src/spec/i686_pc_windows_msvc.rs
+++ b/compiler/rustc_target/src/spec/i686_pc_windows_msvc.rs
@@ -14,10 +14,10 @@ pub fn target() -> Target {
         // https://docs.microsoft.com/en-us/cpp/build/reference/safeseh-image-has-safe-exception-handlers
         "/SAFESEH".to_string(),
     ];
-    base.pre_link_args.get_mut(&LinkerFlavor::Msvc).unwrap().extend(pre_link_args_msvc.clone());
+    base.pre_link_args.entry(LinkerFlavor::Msvc).or_default().extend(pre_link_args_msvc.clone());
     base.pre_link_args
-        .get_mut(&LinkerFlavor::Lld(LldFlavor::Link))
-        .unwrap()
+        .entry(LinkerFlavor::Lld(LldFlavor::Link))
+        .or_default()
         .extend(pre_link_args_msvc);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/i686_unknown_freebsd.rs b/compiler/rustc_target/src/spec/i686_unknown_freebsd.rs
index fc425babb69..a26cabdc90a 100644
--- a/compiler/rustc_target/src/spec/i686_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/i686_unknown_freebsd.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
-    let pre_link_args = base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap();
+    let pre_link_args = base.pre_link_args.entry(LinkerFlavor::Gcc).or_default();
     pre_link_args.push("-m32".to_string());
     pre_link_args.push("-Wl,-znotext".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
diff --git a/compiler/rustc_target/src/spec/i686_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/i686_unknown_linux_gnu.rs
index fe1e6a4299d..633e8da0ccb 100644
--- a/compiler/rustc_target/src/spec/i686_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/i686_unknown_linux_gnu.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/i686_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/i686_unknown_linux_musl.rs
index 623fd1b9ae8..8bcd261e4df 100644
--- a/compiler/rustc_target/src/spec/i686_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/i686_unknown_linux_musl.rs
@@ -4,8 +4,8 @@ pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-Wl,-melf_i386".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-Wl,-melf_i386".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     // The unwinder used by i686-unknown-linux-musl, the LLVM libunwind
diff --git a/compiler/rustc_target/src/spec/i686_unknown_netbsd.rs b/compiler/rustc_target/src/spec/i686_unknown_netbsd.rs
index c4d11bfb13e..e020264ad7a 100644
--- a/compiler/rustc_target/src/spec/i686_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/i686_unknown_netbsd.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/i686_unknown_openbsd.rs b/compiler/rustc_target/src/spec/i686_unknown_openbsd.rs
index fdaaf6c741e..86448cb9115 100644
--- a/compiler/rustc_target/src/spec/i686_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/i686_unknown_openbsd.rs
@@ -4,8 +4,8 @@ pub fn target() -> Target {
     let mut base = super::openbsd_base::opts();
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-fuse-ld=lld".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-fuse-ld=lld".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/i686_uwp_windows_gnu.rs b/compiler/rustc_target/src/spec/i686_uwp_windows_gnu.rs
index a3de93efb78..426df59882d 100644
--- a/compiler/rustc_target/src/spec/i686_uwp_windows_gnu.rs
+++ b/compiler/rustc_target/src/spec/i686_uwp_windows_gnu.rs
@@ -11,8 +11,8 @@ pub fn target() -> Target {
     // Mark all dynamic libraries and executables as compatible with the larger 4GiB address
     // space available to x86 Windows binaries on x86_64.
     base.pre_link_args
-        .get_mut(&LinkerFlavor::Gcc)
-        .unwrap()
+        .entry(LinkerFlavor::Gcc)
+        .or_default()
         .push("-Wl,--large-address-aware".to_string());
 
     Target {
diff --git a/compiler/rustc_target/src/spec/i686_wrs_vxworks.rs b/compiler/rustc_target/src/spec/i686_wrs_vxworks.rs
index ec8a2493b4e..e596eca86b0 100644
--- a/compiler/rustc_target/src/spec/i686_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/i686_wrs_vxworks.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/l4re_base.rs b/compiler/rustc_target/src/spec/l4re_base.rs
index 660fae5f5c7..db6b74eff6d 100644
--- a/compiler/rustc_target/src/spec/l4re_base.rs
+++ b/compiler/rustc_target/src/spec/l4re_base.rs
@@ -1,4 +1,4 @@
-use crate::spec::{LinkArgs, LinkerFlavor, PanicStrategy, TargetOptions};
+use crate::spec::{LinkerFlavor, PanicStrategy, TargetOptions};
 //use std::process::Command;
 
 // Use GCC to locate code for crt* libraries from the host, not from L4Re. Note
@@ -13,18 +13,13 @@ use crate::spec::{LinkArgs, LinkerFlavor, PanicStrategy, TargetOptions};
 //}
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(LinkerFlavor::Gcc, vec![]);
-
     TargetOptions {
         os: "l4re".to_string(),
         env: "uclibc".to_string(),
         linker_flavor: LinkerFlavor::Ld,
         executables: true,
-        has_elf_tls: false,
         panic_strategy: PanicStrategy::Abort,
         linker: Some("ld".to_string()),
-        pre_link_args: args,
         os_family: Some("unix".to_string()),
         ..Default::default()
     }
diff --git a/compiler/rustc_target/src/spec/linux_base.rs b/compiler/rustc_target/src/spec/linux_base.rs
index 0631644ad63..eeefd056e4b 100644
--- a/compiler/rustc_target/src/spec/linux_base.rs
+++ b/compiler/rustc_target/src/spec/linux_base.rs
@@ -1,23 +1,6 @@
-use crate::spec::{LinkArgs, LinkerFlavor, RelroLevel, TargetOptions};
+use crate::spec::{RelroLevel, TargetOptions};
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(
-        LinkerFlavor::Gcc,
-        vec![
-            // We want to be able to strip as much executable code as possible
-            // from the linker command line, and this flag indicates to the
-            // linker that it can avoid linking in dynamic libraries that don't
-            // actually satisfy any symbols up to that point (as with many other
-            // resolutions the linker does). This option only applies to all
-            // following libraries so we're sure to pass it as one of the first
-            // arguments.
-            "-Wl,--as-needed".to_string(),
-            // Always enable NX protection when it is available
-            "-Wl,-z,noexecstack".to_string(),
-        ],
-    );
-
     TargetOptions {
         os: "linux".to_string(),
         dynamic_linking: true,
@@ -25,7 +8,6 @@ pub fn opts() -> TargetOptions {
         os_family: Some("unix".to_string()),
         linker_is_gnu: true,
         has_rpath: true,
-        pre_link_args: args,
         position_independent_executables: true,
         relro_level: RelroLevel::Full,
         has_elf_tls: true,
diff --git a/compiler/rustc_target/src/spec/linux_kernel_base.rs b/compiler/rustc_target/src/spec/linux_kernel_base.rs
index 52201568953..d17d729c289 100644
--- a/compiler/rustc_target/src/spec/linux_kernel_base.rs
+++ b/compiler/rustc_target/src/spec/linux_kernel_base.rs
@@ -1,14 +1,6 @@
-use crate::spec::{
-    LinkArgs, LinkerFlavor, PanicStrategy, RelocModel, RelroLevel, StackProbeType, TargetOptions,
-};
+use crate::spec::{PanicStrategy, RelocModel, RelroLevel, StackProbeType, TargetOptions};
 
 pub fn opts() -> TargetOptions {
-    let mut pre_link_args = LinkArgs::new();
-    pre_link_args.insert(
-        LinkerFlavor::Gcc,
-        vec!["-Wl,--as-needed".to_string(), "-Wl,-z,noexecstack".to_string()],
-    );
-
     TargetOptions {
         env: "gnu".to_string(),
         disable_redzone: true,
@@ -20,7 +12,6 @@ pub fn opts() -> TargetOptions {
         needs_plt: true,
         relro_level: RelroLevel::Full,
         relocation_model: RelocModel::Static,
-        pre_link_args,
 
         ..Default::default()
     }
diff --git a/compiler/rustc_target/src/spec/mipsel_unknown_none.rs b/compiler/rustc_target/src/spec/mipsel_unknown_none.rs
index 0f9d3c3de15..110c8dd80ea 100644
--- a/compiler/rustc_target/src/spec/mipsel_unknown_none.rs
+++ b/compiler/rustc_target/src/spec/mipsel_unknown_none.rs
@@ -23,10 +23,12 @@ pub fn target() -> Target {
             panic_strategy: PanicStrategy::Abort,
             relocation_model: RelocModel::Static,
             unsupported_abis: vec![
-                Abi::Stdcall,
+                Abi::Stdcall { unwind: false },
+                Abi::Stdcall { unwind: true },
                 Abi::Fastcall,
                 Abi::Vectorcall,
-                Abi::Thiscall,
+                Abi::Thiscall { unwind: false },
+                Abi::Thiscall { unwind: true },
                 Abi::Win64,
                 Abi::SysV64,
             ],
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 72b2d089ebc..490239ee615 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -37,6 +37,7 @@
 use crate::abi::Endian;
 use crate::spec::abi::{lookup as lookup_abi, Abi};
 use crate::spec::crt_objects::{CrtObjects, CrtObjectsFallback};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_serialize::json::{Json, ToJson};
 use rustc_span::symbol::{sym, Symbol};
 use std::collections::BTreeMap;
@@ -78,7 +79,7 @@ mod solaris_base;
 mod thumb_base;
 mod uefi_msvc_base;
 mod vxworks_base;
-mod wasm32_base;
+mod wasm_base;
 mod windows_gnu_base;
 mod windows_msvc_base;
 mod windows_uwp_gnu_base;
@@ -511,38 +512,6 @@ impl fmt::Display for SplitDebuginfo {
     }
 }
 
-macro_rules! supported_targets {
-    ( $(($( $triple:literal, )+ $module:ident ),)+ ) => {
-        $(mod $module;)+
-
-        /// List of supported targets
-        pub const TARGETS: &[&str] = &[$($($triple),+),+];
-
-        fn load_builtin(target: &str) -> Option<Target> {
-            let mut t = match target {
-                $( $($triple)|+ => $module::target(), )+
-                _ => return None,
-            };
-            t.is_builtin = true;
-            debug!("got builtin target: {:?}", t);
-            Some(t)
-        }
-
-        #[cfg(test)]
-        mod tests {
-            mod tests_impl;
-
-            // Cannot put this into a separate file without duplication, make an exception.
-            $(
-                #[test] // `#[test]`
-                fn $module() {
-                    tests_impl::test_target(super::$module::target());
-                }
-            )+
-        }
-    };
-}
-
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum StackProbeType {
     /// Don't emit any stack probes.
@@ -620,6 +589,117 @@ impl ToJson for StackProbeType {
     }
 }
 
+bitflags::bitflags! {
+    #[derive(Default, Encodable, Decodable)]
+    pub struct SanitizerSet: u8 {
+        const ADDRESS = 1 << 0;
+        const LEAK    = 1 << 1;
+        const MEMORY  = 1 << 2;
+        const THREAD  = 1 << 3;
+        const HWADDRESS = 1 << 4;
+    }
+}
+
+impl SanitizerSet {
+    /// Return sanitizer's name
+    ///
+    /// Returns none if the flags is a set of sanitizers numbering not exactly one.
+    fn as_str(self) -> Option<&'static str> {
+        Some(match self {
+            SanitizerSet::ADDRESS => "address",
+            SanitizerSet::LEAK => "leak",
+            SanitizerSet::MEMORY => "memory",
+            SanitizerSet::THREAD => "thread",
+            SanitizerSet::HWADDRESS => "hwaddress",
+            _ => return None,
+        })
+    }
+}
+
+/// Formats a sanitizer set as a comma separated list of sanitizers' names.
+impl fmt::Display for SanitizerSet {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut first = true;
+        for s in *self {
+            let name = s.as_str().unwrap_or_else(|| panic!("unrecognized sanitizer {:?}", s));
+            if !first {
+                f.write_str(", ")?;
+            }
+            f.write_str(name)?;
+            first = false;
+        }
+        Ok(())
+    }
+}
+
+impl IntoIterator for SanitizerSet {
+    type Item = SanitizerSet;
+    type IntoIter = std::vec::IntoIter<SanitizerSet>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        [
+            SanitizerSet::ADDRESS,
+            SanitizerSet::LEAK,
+            SanitizerSet::MEMORY,
+            SanitizerSet::THREAD,
+            SanitizerSet::HWADDRESS,
+        ]
+        .iter()
+        .copied()
+        .filter(|&s| self.contains(s))
+        .collect::<Vec<_>>()
+        .into_iter()
+    }
+}
+
+impl<CTX> HashStable<CTX> for SanitizerSet {
+    fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
+        self.bits().hash_stable(ctx, hasher);
+    }
+}
+
+impl ToJson for SanitizerSet {
+    fn to_json(&self) -> Json {
+        self.into_iter()
+            .map(|v| Some(v.as_str()?.to_json()))
+            .collect::<Option<Vec<_>>>()
+            .unwrap_or(Vec::new())
+            .to_json()
+    }
+}
+
+macro_rules! supported_targets {
+    ( $(($( $triple:literal, )+ $module:ident ),)+ ) => {
+        $(mod $module;)+
+
+        /// List of supported targets
+        pub const TARGETS: &[&str] = &[$($($triple),+),+];
+
+        fn load_builtin(target: &str) -> Option<Target> {
+            let mut t = match target {
+                $( $($triple)|+ => $module::target(), )+
+                _ => return None,
+            };
+            t.is_builtin = true;
+            debug!("got builtin target: {:?}", t);
+            Some(t)
+        }
+
+        #[cfg(test)]
+        mod tests {
+            mod tests_impl;
+
+            // Cannot put this into a separate file without duplication, make an exception.
+            $(
+                #[test] // `#[test]`
+                fn $module() {
+                    tests_impl::test_target(super::$module::target());
+                }
+            )+
+        }
+    };
+}
+
 supported_targets! {
     ("x86_64-unknown-linux-gnu", x86_64_unknown_linux_gnu),
     ("x86_64-unknown-linux-gnux32", x86_64_unknown_linux_gnux32),
@@ -679,7 +759,7 @@ supported_targets! {
     ("thumbv7neon-linux-androideabi", thumbv7neon_linux_androideabi),
     ("aarch64-linux-android", aarch64_linux_android),
 
-    ("x86_64-linux-kernel", x86_64_linux_kernel),
+    ("x86_64-unknown-none-linuxkernel", x86_64_unknown_none_linuxkernel),
 
     ("aarch64-unknown-freebsd", aarch64_unknown_freebsd),
     ("armv6-unknown-freebsd", armv6_unknown_freebsd),
@@ -694,6 +774,7 @@ supported_targets! {
     ("i686-unknown-openbsd", i686_unknown_openbsd),
     ("sparc64-unknown-openbsd", sparc64_unknown_openbsd),
     ("x86_64-unknown-openbsd", x86_64_unknown_openbsd),
+    ("powerpc-unknown-openbsd", powerpc_unknown_openbsd),
 
     ("aarch64-unknown-netbsd", aarch64_unknown_netbsd),
     ("armv6-unknown-netbsd-eabihf", armv6_unknown_netbsd_eabihf),
@@ -761,6 +842,7 @@ supported_targets! {
     ("wasm32-unknown-emscripten", wasm32_unknown_emscripten),
     ("wasm32-unknown-unknown", wasm32_unknown_unknown),
     ("wasm32-wasi", wasm32_wasi),
+    ("wasm64-unknown-unknown", wasm64_unknown_unknown),
 
     ("thumbv6m-none-eabi", thumbv6m_none_eabi),
     ("thumbv7m-none-eabi", thumbv7m_none_eabi),
@@ -777,7 +859,8 @@ supported_targets! {
 
     ("aarch64-unknown-hermit", aarch64_unknown_hermit),
     ("x86_64-unknown-hermit", x86_64_unknown_hermit),
-    ("x86_64-unknown-hermit-kernel", x86_64_unknown_hermit_kernel),
+
+    ("x86_64-unknown-none-hermitkernel", x86_64_unknown_none_hermitkernel),
 
     ("riscv32i-unknown-none-elf", riscv32i_unknown_none_elf),
     ("riscv32imc-unknown-none-elf", riscv32imc_unknown_none_elf),
@@ -994,6 +1077,8 @@ pub struct TargetOptions {
     pub is_like_emscripten: bool,
     /// Whether the target toolchain is like Fuchsia's.
     pub is_like_fuchsia: bool,
+    /// Whether a target toolchain is like WASM.
+    pub is_like_wasm: bool,
     /// Version of DWARF to use if not using the default.
     /// Useful because some platforms (osx, bsd) only want up to DWARF2.
     pub dwarf_version: Option<u32>,
@@ -1109,6 +1194,10 @@ pub struct TargetOptions {
     /// unwinders.
     pub requires_uwtable: bool,
 
+    /// Whether or not to emit `uwtable` attributes on functions if `-C force-unwind-tables`
+    /// is not specified and `uwtable` is not required on this target.
+    pub default_uwtable: bool,
+
     /// Whether or not SIMD types are passed by reference in the Rust ABI,
     /// typically required if a target can be compiled with a mixed set of
     /// target features. This is `true` by default, and `false` for targets like
@@ -1158,6 +1247,16 @@ pub struct TargetOptions {
     /// How to handle split debug information, if at all. Specifying `None` has
     /// target-specific meaning.
     pub split_debuginfo: SplitDebuginfo,
+
+    /// The sanitizers supported by this target
+    ///
+    /// Note that the support here is at a codegen level. If the machine code with sanitizer
+    /// enabled can generated on this target, but the necessary supporting libraries are not
+    /// distributed with the target, the sanitizer should still appear in this list for the target.
+    pub supported_sanitizers: SanitizerSet,
+
+    /// If present it's a default value to use for adjusting the C ABI.
+    pub default_adjusted_cabi: Option<Abi>,
 }
 
 impl Default for TargetOptions {
@@ -1202,6 +1301,7 @@ impl Default for TargetOptions {
             is_like_emscripten: false,
             is_like_msvc: false,
             is_like_fuchsia: false,
+            is_like_wasm: false,
             dwarf_version: None,
             linker_is_gnu: false,
             allows_weak_linkage: true,
@@ -1246,6 +1346,7 @@ impl Default for TargetOptions {
             default_hidden_visibility: false,
             emit_debug_gdb_scripts: true,
             requires_uwtable: false,
+            default_uwtable: false,
             simd_types_indirect: true,
             limit_rdylib_exports: true,
             override_export_symbols: None,
@@ -1258,6 +1359,8 @@ impl Default for TargetOptions {
             eh_frame_header: true,
             has_thumb_interworking: false,
             split_debuginfo: SplitDebuginfo::Off,
+            supported_sanitizers: SanitizerSet::empty(),
+            default_adjusted_cabi: None,
         }
     }
 }
@@ -1282,26 +1385,36 @@ impl Target {
     /// Given a function ABI, turn it into the correct ABI for this target.
     pub fn adjust_abi(&self, abi: Abi) -> Abi {
         match abi {
-            Abi::System => {
+            Abi::System { unwind } => {
                 if self.is_like_windows && self.arch == "x86" {
-                    Abi::Stdcall
+                    Abi::Stdcall { unwind }
                 } else {
-                    Abi::C
+                    Abi::C { unwind }
                 }
             }
             // These ABI kinds are ignored on non-x86 Windows targets.
             // See https://docs.microsoft.com/en-us/cpp/cpp/argument-passing-and-naming-conventions
             // and the individual pages for __stdcall et al.
-            Abi::Stdcall | Abi::Fastcall | Abi::Vectorcall | Abi::Thiscall => {
-                if self.is_like_windows && self.arch != "x86" { Abi::C } else { abi }
+            Abi::Stdcall { unwind } | Abi::Thiscall { unwind } => {
+                if self.is_like_windows && self.arch != "x86" { Abi::C { unwind } } else { abi }
+            }
+            Abi::Fastcall | Abi::Vectorcall => {
+                if self.is_like_windows && self.arch != "x86" {
+                    Abi::C { unwind: false }
+                } else {
+                    abi
+                }
             }
             Abi::EfiApi => {
                 if self.arch == "x86_64" {
                     Abi::Win64
                 } else {
-                    Abi::C
+                    Abi::C { unwind: false }
                 }
             }
+
+            Abi::C { unwind } => self.default_adjusted_cabi.unwrap_or(Abi::C { unwind }),
+
             abi => abi,
         }
     }
@@ -1333,8 +1446,8 @@ impl Target {
 
         let get_req_field = |name: &str| {
             obj.find(name)
-                .map(|s| s.as_string())
-                .and_then(|os| os.map(|s| s.to_string()))
+                .and_then(Json::as_string)
+                .map(str::to_string)
                 .ok_or_else(|| format!("Field {} in target specification is required", name))
         };
 
@@ -1537,6 +1650,24 @@ impl Target {
                     )),
                 }).unwrap_or(Ok(()))
             } );
+            ($key_name:ident, SanitizerSet) => ( {
+                let name = (stringify!($key_name)).replace("_", "-");
+                obj.find(&name[..]).and_then(|o| o.as_array()).and_then(|a| {
+                    for s in a {
+                        base.$key_name |= match s.as_string() {
+                            Some("address") => SanitizerSet::ADDRESS,
+                            Some("leak") => SanitizerSet::LEAK,
+                            Some("memory") => SanitizerSet::MEMORY,
+                            Some("thread") => SanitizerSet::THREAD,
+                            Some("hwaddress") => SanitizerSet::HWADDRESS,
+                            Some(s) => return Some(Err(format!("unknown sanitizer {}", s))),
+                            _ => return Some(Err(format!("not a string: {:?}", s))),
+                        };
+                    }
+                    Some(Ok(()))
+                }).unwrap_or(Ok(()))
+            } );
+
             ($key_name:ident, crt_objects_fallback) => ( {
                 let name = (stringify!($key_name)).replace("_", "-");
                 obj.find(&name[..]).and_then(|o| o.as_string().and_then(|s| {
@@ -1618,6 +1749,16 @@ impl Target {
                     }
                 }
             } );
+            ($key_name:ident, Option<Abi>) => ( {
+                let name = (stringify!($key_name)).replace("_", "-");
+                obj.find(&name[..]).and_then(|o| o.as_string().and_then(|s| {
+                    match lookup_abi(s) {
+                        Some(abi) => base.$key_name = Some(abi),
+                        _ => return Some(Err(format!("'{}' is not a valid value for abi", s))),
+                    }
+                    Some(Ok(()))
+                })).unwrap_or(Ok(()))
+            } );
         }
 
         if let Some(s) = obj.find("target-endian").and_then(Json::as_string) {
@@ -1669,6 +1810,7 @@ impl Target {
         key!(is_like_msvc, bool);
         key!(is_like_emscripten, bool);
         key!(is_like_fuchsia, bool);
+        key!(is_like_wasm, bool);
         key!(dwarf_version, Option<u32>);
         key!(linker_is_gnu, bool);
         key!(allows_weak_linkage, bool);
@@ -1702,6 +1844,7 @@ impl Target {
         key!(default_hidden_visibility, bool);
         key!(emit_debug_gdb_scripts, bool);
         key!(requires_uwtable, bool);
+        key!(default_uwtable, bool);
         key!(simd_types_indirect, bool);
         key!(limit_rdylib_exports, bool);
         key!(override_export_symbols, opt_list);
@@ -1714,6 +1857,8 @@ impl Target {
         key!(eh_frame_header, bool);
         key!(has_thumb_interworking, bool);
         key!(split_debuginfo, SplitDebuginfo)?;
+        key!(supported_sanitizers, SanitizerSet)?;
+        key!(default_adjusted_cabi, Option<Abi>)?;
 
         // NB: The old name is deprecated, but support for it is retained for
         // compatibility.
@@ -1914,6 +2059,7 @@ impl ToJson for Target {
         target_option_val!(is_like_msvc);
         target_option_val!(is_like_emscripten);
         target_option_val!(is_like_fuchsia);
+        target_option_val!(is_like_wasm);
         target_option_val!(dwarf_version);
         target_option_val!(linker_is_gnu);
         target_option_val!(allows_weak_linkage);
@@ -1947,6 +2093,7 @@ impl ToJson for Target {
         target_option_val!(default_hidden_visibility);
         target_option_val!(emit_debug_gdb_scripts);
         target_option_val!(requires_uwtable);
+        target_option_val!(default_uwtable);
         target_option_val!(simd_types_indirect);
         target_option_val!(limit_rdylib_exports);
         target_option_val!(override_export_symbols);
@@ -1959,6 +2106,11 @@ impl ToJson for Target {
         target_option_val!(eh_frame_header);
         target_option_val!(has_thumb_interworking);
         target_option_val!(split_debuginfo);
+        target_option_val!(supported_sanitizers);
+
+        if let Some(abi) = self.default_adjusted_cabi {
+            d.insert("default-adjusted-cabi".to_string(), Abi::name(abi).to_json());
+        }
 
         if default.unsupported_abis != self.unsupported_abis {
             d.insert(
diff --git a/compiler/rustc_target/src/spec/msvc_base.rs b/compiler/rustc_target/src/spec/msvc_base.rs
index 39c0d5f0bb4..4ed7685ca07 100644
--- a/compiler/rustc_target/src/spec/msvc_base.rs
+++ b/compiler/rustc_target/src/spec/msvc_base.rs
@@ -5,13 +5,6 @@ pub fn opts() -> TargetOptions {
         // Suppress the verbose logo and authorship debugging output, which would needlessly
         // clog any log files.
         "/NOLOGO".to_string(),
-        // Tell the compiler that non-code sections can be marked as non-executable,
-        // including stack pages.
-        // UEFI is fully compatible to non-executable data pages.
-        // In fact, firmware might enforce this, so we better let the linker know about this,
-        // so it will fail if the compiler ever tries placing code on the stack
-        // (e.g., trampoline constructs and alike).
-        "/NXCOMPAT".to_string(),
     ];
     let mut pre_link_args = LinkArgs::new();
     pre_link_args.insert(LinkerFlavor::Msvc, pre_link_args_msvc.clone());
diff --git a/compiler/rustc_target/src/spec/netbsd_base.rs b/compiler/rustc_target/src/spec/netbsd_base.rs
index a77d60bd9d7..680cd60788b 100644
--- a/compiler/rustc_target/src/spec/netbsd_base.rs
+++ b/compiler/rustc_target/src/spec/netbsd_base.rs
@@ -1,18 +1,6 @@
-use crate::spec::{LinkArgs, LinkerFlavor, RelroLevel, TargetOptions};
+use crate::spec::{RelroLevel, TargetOptions};
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(
-        LinkerFlavor::Gcc,
-        vec![
-            // GNU-style linkers will use this to omit linking to libraries
-            // which don't actually fulfill any relocations, but only for
-            // libraries which follow this flag.  Thus, use it before
-            // specifying libraries to link to.
-            "-Wl,--as-needed".to_string(),
-        ],
-    );
-
     TargetOptions {
         os: "netbsd".to_string(),
         dynamic_linking: true,
@@ -21,7 +9,6 @@ pub fn opts() -> TargetOptions {
         linker_is_gnu: true,
         no_default_libraries: false,
         has_rpath: true,
-        pre_link_args: args,
         position_independent_executables: true,
         relro_level: RelroLevel::Full,
         use_ctors_section: true,
diff --git a/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs b/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs
index 3c9c7d578fb..15d8e4843f9 100644
--- a/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs
+++ b/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs
@@ -49,10 +49,12 @@ pub fn target() -> Target {
             // create the tests for this.
             unsupported_abis: vec![
                 Abi::Cdecl,
-                Abi::Stdcall,
+                Abi::Stdcall { unwind: false },
+                Abi::Stdcall { unwind: true },
                 Abi::Fastcall,
                 Abi::Vectorcall,
-                Abi::Thiscall,
+                Abi::Thiscall { unwind: false },
+                Abi::Thiscall { unwind: true },
                 Abi::Aapcs,
                 Abi::Win64,
                 Abi::SysV64,
diff --git a/compiler/rustc_target/src/spec/openbsd_base.rs b/compiler/rustc_target/src/spec/openbsd_base.rs
index 2b40a1ed945..a6fd01ab110 100644
--- a/compiler/rustc_target/src/spec/openbsd_base.rs
+++ b/compiler/rustc_target/src/spec/openbsd_base.rs
@@ -1,20 +1,6 @@
-use crate::spec::{LinkArgs, LinkerFlavor, RelroLevel, TargetOptions};
+use crate::spec::{RelroLevel, TargetOptions};
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(
-        LinkerFlavor::Gcc,
-        vec![
-            // GNU-style linkers will use this to omit linking to libraries
-            // which don't actually fulfill any relocations, but only for
-            // libraries which follow this flag.  Thus, use it before
-            // specifying libraries to link to.
-            "-Wl,--as-needed".to_string(),
-            // Always enable NX protection when it is available
-            "-Wl,-z,noexecstack".to_string(),
-        ],
-    );
-
     TargetOptions {
         os: "openbsd".to_string(),
         dynamic_linking: true,
@@ -23,7 +9,6 @@ pub fn opts() -> TargetOptions {
         linker_is_gnu: true,
         has_rpath: true,
         abi_return_struct_as_int: true,
-        pre_link_args: args,
         position_independent_executables: true,
         eliminate_frame_pointer: false, // FIXME 43575
         relro_level: RelroLevel::Full,
diff --git a/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs
index 3dddeb1129c..b3d6b7c6107 100644
--- a/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs
@@ -4,7 +4,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     base.cpu = "ppc64".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs
index 9db880b0e53..559a1a40868 100644
--- a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs
@@ -4,7 +4,7 @@ use crate::spec::{LinkerFlavor, RelroLevel, Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "ppc64".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     // ld.so in at least RHEL6 on ppc64 has a bug related to BIND_NOW, so only enable partial RELRO
diff --git a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs
index 8767f86b00b..f1190b159ab 100644
--- a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs
@@ -4,7 +4,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "ppc64".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs
index 2f28a856247..3ebc5469e0a 100644
--- a/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs
@@ -4,7 +4,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.cpu = "ppc64".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs
index 4cbd9976508..76f70e474f0 100644
--- a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "ppc64le".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs
index efdc9ad7517..42c49103b3b 100644
--- a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "ppc64le".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs
index 70dd0b2aee6..21ffdd2d160 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
     base.max_atomic_width = Some(32);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs
index 66118b74955..8d8f746f97f 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-mspe".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-mspe".to_string());
     base.max_atomic_width = Some(32);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs
index 679a3a2f6aa..9633705db6d 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
     base.max_atomic_width = Some(32);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs b/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs
index 1245098329a..4cc5224fae3 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
     base.max_atomic_width = Some(32);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs b/compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs
new file mode 100644
index 00000000000..c17183faa7a
--- /dev/null
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs
@@ -0,0 +1,16 @@
+use crate::abi::Endian;
+use crate::spec::Target;
+
+pub fn target() -> Target {
+    let mut base = super::openbsd_base::opts();
+    base.endian = Endian::Big;
+    base.max_atomic_width = Some(32);
+
+    Target {
+        llvm_target: "powerpc-unknown-openbsd".to_string(),
+        pointer_width: 32,
+        data_layout: "E-m:e-p:32:32-i64:64-n32".to_string(),
+        arch: "powerpc".to_string(),
+        options: base,
+    }
+}
diff --git a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs
index bb943a8825c..2f0a6ca44a0 100644
--- a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs
@@ -3,8 +3,8 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("--secure-plt".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("--secure-plt".to_string());
     base.max_atomic_width = Some(32);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs
index 4b4f118ba49..215f1a36227 100644
--- a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs
+++ b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs
@@ -3,8 +3,8 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-mspe".to_string());
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("--secure-plt".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-mspe".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("--secure-plt".to_string());
     base.max_atomic_width = Some(32);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/redox_base.rs b/compiler/rustc_target/src/spec/redox_base.rs
index 5ef705878a8..0afb4a72ac1 100644
--- a/compiler/rustc_target/src/spec/redox_base.rs
+++ b/compiler/rustc_target/src/spec/redox_base.rs
@@ -1,23 +1,6 @@
-use crate::spec::{LinkArgs, LinkerFlavor, RelroLevel, TargetOptions};
+use crate::spec::{RelroLevel, TargetOptions};
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(
-        LinkerFlavor::Gcc,
-        vec![
-            // We want to be able to strip as much executable code as possible
-            // from the linker command line, and this flag indicates to the
-            // linker that it can avoid linking in dynamic libraries that don't
-            // actually satisfy any symbols up to that point (as with many other
-            // resolutions the linker does). This option only applies to all
-            // following libraries so we're sure to pass it as one of the first
-            // arguments.
-            "-Wl,--as-needed".to_string(),
-            // Always enable NX protection when it is available
-            "-Wl,-z,noexecstack".to_string(),
-        ],
-    );
-
     TargetOptions {
         os: "redox".to_string(),
         env: "relibc".to_string(),
@@ -26,7 +9,6 @@ pub fn opts() -> TargetOptions {
         os_family: Some("unix".to_string()),
         linker_is_gnu: true,
         has_rpath: true,
-        pre_link_args: args,
         position_independent_executables: true,
         relro_level: RelroLevel::Full,
         has_elf_tls: true,
diff --git a/compiler/rustc_target/src/spec/riscv32i_unknown_none_elf.rs b/compiler/rustc_target/src/spec/riscv32i_unknown_none_elf.rs
index a31a08a8cf9..88a22f25ff4 100644
--- a/compiler/rustc_target/src/spec/riscv32i_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/riscv32i_unknown_none_elf.rs
@@ -14,7 +14,6 @@ pub fn target() -> Target {
             cpu: "generic-rv32".to_string(),
             max_atomic_width: Some(0),
             atomic_cas: false,
-            features: String::new(),
             executables: true,
             panic_strategy: PanicStrategy::Abort,
             relocation_model: RelocModel::Static,
diff --git a/compiler/rustc_target/src/spec/riscv32imac_unknown_none_elf.rs b/compiler/rustc_target/src/spec/riscv32imac_unknown_none_elf.rs
index 2ee53fdc401..b406eec1e75 100644
--- a/compiler/rustc_target/src/spec/riscv32imac_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/riscv32imac_unknown_none_elf.rs
@@ -13,7 +13,6 @@ pub fn target() -> Target {
             linker: Some("rust-lld".to_string()),
             cpu: "generic-rv32".to_string(),
             max_atomic_width: Some(32),
-            atomic_cas: true,
             features: "+m,+a,+c".to_string(),
             executables: true,
             panic_strategy: PanicStrategy::Abort,
diff --git a/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs b/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs
index 33a785fdfee..481bce05a08 100644
--- a/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs
@@ -11,9 +11,9 @@ pub fn target() -> Target {
         options: TargetOptions {
             linker_flavor: LinkerFlavor::Lld(LldFlavor::Ld),
             linker: Some("rust-lld".to_string()),
+            llvm_abiname: "lp64d".to_string(),
             cpu: "generic-rv64".to_string(),
             max_atomic_width: Some(64),
-            atomic_cas: true,
             features: "+m,+a,+f,+d,+c".to_string(),
             executables: true,
             panic_strategy: PanicStrategy::Abort,
diff --git a/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs b/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs
index 908367ee200..3e4afd446dd 100644
--- a/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs
@@ -13,7 +13,6 @@ pub fn target() -> Target {
             linker: Some("rust-lld".to_string()),
             cpu: "generic-rv64".to_string(),
             max_atomic_width: Some(64),
-            atomic_cas: true,
             features: "+m,+a,+c".to_string(),
             executables: true,
             panic_strategy: PanicStrategy::Abort,
diff --git a/compiler/rustc_target/src/spec/riscv_base.rs b/compiler/rustc_target/src/spec/riscv_base.rs
index 64cf890037e..5bcbb2e621b 100644
--- a/compiler/rustc_target/src/spec/riscv_base.rs
+++ b/compiler/rustc_target/src/spec/riscv_base.rs
@@ -5,10 +5,12 @@ use crate::spec::abi::Abi;
 pub fn unsupported_abis() -> Vec<Abi> {
     vec![
         Abi::Cdecl,
-        Abi::Stdcall,
+        Abi::Stdcall { unwind: false },
+        Abi::Stdcall { unwind: true },
         Abi::Fastcall,
         Abi::Vectorcall,
-        Abi::Thiscall,
+        Abi::Thiscall { unwind: false },
+        Abi::Thiscall { unwind: true },
         Abi::Aapcs,
         Abi::Win64,
         Abi::SysV64,
diff --git a/compiler/rustc_target/src/spec/sparc64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/sparc64_unknown_netbsd.rs
index 7d685c83100..b4286dfd88f 100644
--- a/compiler/rustc_target/src/spec/sparc64_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/sparc64_unknown_netbsd.rs
@@ -4,7 +4,7 @@ use crate::spec::{LinkerFlavor, Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
     base.cpu = "v9".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/sparc64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/sparc64_unknown_openbsd.rs
index 63b13fad4f7..9732983161f 100644
--- a/compiler/rustc_target/src/spec/sparc64_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/sparc64_unknown_openbsd.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     let mut base = super::openbsd_base::opts();
     base.endian = Endian::Big;
     base.cpu = "v9".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.max_atomic_width = Some(64);
 
     Target {
diff --git a/compiler/rustc_target/src/spec/sparc_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/sparc_unknown_linux_gnu.rs
index 9e8fbff81c5..1fd4cadfffc 100644
--- a/compiler/rustc_target/src/spec/sparc_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/sparc_unknown_linux_gnu.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.endian = Endian::Big;
     base.cpu = "v9".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-mv8plus".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-mv8plus".to_string());
 
     Target {
         llvm_target: "sparc-unknown-linux-gnu".to_string(),
diff --git a/compiler/rustc_target/src/spec/tests/tests_impl.rs b/compiler/rustc_target/src/spec/tests/tests_impl.rs
index 9ec8467e0ac..f4de8bc0a58 100644
--- a/compiler/rustc_target/src/spec/tests/tests_impl.rs
+++ b/compiler/rustc_target/src/spec/tests/tests_impl.rs
@@ -50,6 +50,7 @@ impl Target {
     // and you certainly want "unknown" for the OS name.
     fn can_use_os_unknown(&self) -> bool {
         self.llvm_target == "wasm32-unknown-unknown"
+            || self.llvm_target == "wasm64-unknown-unknown"
             || (self.env == "sgx" && self.vendor == "fortanix")
     }
 }
diff --git a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
index d87c06d49cb..ef58824f381 100644
--- a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
@@ -45,9 +45,6 @@ pub fn target() -> Target {
 
             main_needs_argc_argv: false,
 
-            // No thread-local storage (just use a static Cell)
-            has_elf_tls: false,
-
             // don't have atomic compare-and-swap
             atomic_cas: false,
             has_thumb_interworking: true,
diff --git a/compiler/rustc_target/src/spec/thumbv7a_pc_windows_msvc.rs b/compiler/rustc_target/src/spec/thumbv7a_pc_windows_msvc.rs
index 8131a6e2ea4..1232daa577f 100644
--- a/compiler/rustc_target/src/spec/thumbv7a_pc_windows_msvc.rs
+++ b/compiler/rustc_target/src/spec/thumbv7a_pc_windows_msvc.rs
@@ -11,10 +11,10 @@ pub fn target() -> Target {
     // where necessary, but this is not the observed behavior.
     // Disabling the LBR optimization works around the issue.
     let pre_link_args_msvc = "/OPT:NOLBR".to_string();
-    base.pre_link_args.get_mut(&LinkerFlavor::Msvc).unwrap().push(pre_link_args_msvc.clone());
+    base.pre_link_args.entry(LinkerFlavor::Msvc).or_default().push(pre_link_args_msvc.clone());
     base.pre_link_args
-        .get_mut(&LinkerFlavor::Lld(LldFlavor::Link))
-        .unwrap()
+        .entry(LinkerFlavor::Lld(LldFlavor::Link))
+        .or_default()
         .push(pre_link_args_msvc);
 
     // FIXME(jordanrh): use PanicStrategy::Unwind when SEH is
@@ -29,7 +29,6 @@ pub fn target() -> Target {
 
         options: TargetOptions {
             features: "+vfp3,+neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             ..base
diff --git a/compiler/rustc_target/src/spec/thumbv7a_uwp_windows_msvc.rs b/compiler/rustc_target/src/spec/thumbv7a_uwp_windows_msvc.rs
index a2c1b6bb90c..e6a59f015c9 100644
--- a/compiler/rustc_target/src/spec/thumbv7a_uwp_windows_msvc.rs
+++ b/compiler/rustc_target/src/spec/thumbv7a_uwp_windows_msvc.rs
@@ -16,7 +16,6 @@ pub fn target() -> Target {
         arch: "arm".to_string(),
         options: TargetOptions {
             features: "+vfp3,+neon".to_string(),
-            cpu: "generic".to_string(),
             unsupported_abis: super::arm_base::unsupported_abis(),
             ..base
         },
diff --git a/compiler/rustc_target/src/spec/thumbv7neon_linux_androideabi.rs b/compiler/rustc_target/src/spec/thumbv7neon_linux_androideabi.rs
index 41fdbc2f0a0..58b0a9d2202 100644
--- a/compiler/rustc_target/src/spec/thumbv7neon_linux_androideabi.rs
+++ b/compiler/rustc_target/src/spec/thumbv7neon_linux_androideabi.rs
@@ -12,7 +12,7 @@ pub fn target() -> Target {
     let mut base = super::android_base::opts();
     base.features = "+v7,+thumb-mode,+thumb2,+vfp3,+neon".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-march=armv7-a".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-march=armv7-a".to_string());
 
     Target {
         llvm_target: "armv7-none-linux-android".to_string(),
diff --git a/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_gnueabihf.rs b/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_gnueabihf.rs
index 352d2468743..12d816d095b 100644
--- a/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_gnueabihf.rs
+++ b/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_gnueabihf.rs
@@ -17,7 +17,6 @@ pub fn target() -> Target {
         options: TargetOptions {
             // Info about features at https://wiki.debian.org/ArmHardFloatPort
             features: "+v7,+thumb-mode,+thumb2,+vfp3,+neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             ..base
diff --git a/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_musleabihf.rs b/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_musleabihf.rs
index a788167aede..020de87147c 100644
--- a/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_musleabihf.rs
+++ b/compiler/rustc_target/src/spec/thumbv7neon_unknown_linux_musleabihf.rs
@@ -21,7 +21,6 @@ pub fn target() -> Target {
         // target.
         options: TargetOptions {
             features: "+v7,+thumb-mode,+thumb2,+vfp3,+neon".to_string(),
-            cpu: "generic".to_string(),
             max_atomic_width: Some(64),
             unsupported_abis: super::arm_base::unsupported_abis(),
             mcount: "\u{1}mcount".to_string(),
diff --git a/compiler/rustc_target/src/spec/uefi_msvc_base.rs b/compiler/rustc_target/src/spec/uefi_msvc_base.rs
index b9ff16bd19f..6b6b6018601 100644
--- a/compiler/rustc_target/src/spec/uefi_msvc_base.rs
+++ b/compiler/rustc_target/src/spec/uefi_msvc_base.rs
@@ -30,10 +30,10 @@ pub fn opts() -> TargetOptions {
         // exit (default for applications).
         "/subsystem:efi_application".to_string(),
     ];
-    base.pre_link_args.get_mut(&LinkerFlavor::Msvc).unwrap().extend(pre_link_args_msvc.clone());
+    base.pre_link_args.entry(LinkerFlavor::Msvc).or_default().extend(pre_link_args_msvc.clone());
     base.pre_link_args
-        .get_mut(&LinkerFlavor::Lld(LldFlavor::Link))
-        .unwrap()
+        .entry(LinkerFlavor::Lld(LldFlavor::Link))
+        .or_default()
         .extend(pre_link_args_msvc);
 
     TargetOptions {
diff --git a/compiler/rustc_target/src/spec/vxworks_base.rs b/compiler/rustc_target/src/spec/vxworks_base.rs
index 70bc9ce3e0e..41c4d7625af 100644
--- a/compiler/rustc_target/src/spec/vxworks_base.rs
+++ b/compiler/rustc_target/src/spec/vxworks_base.rs
@@ -1,21 +1,6 @@
-use crate::spec::{LinkArgs, LinkerFlavor, TargetOptions};
+use crate::spec::TargetOptions;
 
 pub fn opts() -> TargetOptions {
-    let mut args = LinkArgs::new();
-    args.insert(
-        LinkerFlavor::Gcc,
-        vec![
-            // We want to be able to strip as much executable code as possible
-            // from the linker command line, and this flag indicates to the
-            // linker that it can avoid linking in dynamic libraries that don't
-            // actually satisfy any symbols up to that point (as with many other
-            // resolutions the linker does). This option only applies to all
-            // following libraries so we're sure to pass it as one of the first
-            // arguments.
-            "-Wl,--as-needed".to_string(),
-        ],
-    );
-
     TargetOptions {
         os: "vxworks".to_string(),
         env: "gnu".to_string(),
@@ -27,8 +12,6 @@ pub fn opts() -> TargetOptions {
         os_family: Some("unix".to_string()),
         linker_is_gnu: true,
         has_rpath: true,
-        pre_link_args: args,
-        position_independent_executables: false,
         has_elf_tls: true,
         crt_static_default: true,
         crt_static_respected: true,
diff --git a/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs b/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs
index 9f69ce16c21..e028dbaa325 100644
--- a/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs
+++ b/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs
@@ -1,10 +1,10 @@
-use super::wasm32_base;
+use super::wasm_base;
 use super::{LinkArgs, LinkerFlavor, PanicStrategy, Target, TargetOptions};
 
 pub fn target() -> Target {
-    let mut options = wasm32_base::options();
+    let mut options = wasm_base::options();
 
-    let clang_args = options.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap();
+    let clang_args = options.pre_link_args.entry(LinkerFlavor::Gcc).or_default();
 
     // Rust really needs a way for users to specify exports and imports in
     // the source code. --export-dynamic isn't the right tool for this job,
diff --git a/compiler/rustc_target/src/spec/wasm32_unknown_unknown.rs b/compiler/rustc_target/src/spec/wasm32_unknown_unknown.rs
index 5e89ba2520b..834c4dbfc05 100644
--- a/compiler/rustc_target/src/spec/wasm32_unknown_unknown.rs
+++ b/compiler/rustc_target/src/spec/wasm32_unknown_unknown.rs
@@ -10,14 +10,26 @@
 //! This target is more or less managed by the Rust and WebAssembly Working
 //! Group nowadays at <https://github.com/rustwasm>.
 
-use super::wasm32_base;
+use super::wasm_base;
 use super::{LinkerFlavor, LldFlavor, Target};
+use crate::spec::abi::Abi;
 
 pub fn target() -> Target {
-    let mut options = wasm32_base::options();
+    let mut options = wasm_base::options();
     options.os = "unknown".to_string();
     options.linker_flavor = LinkerFlavor::Lld(LldFlavor::Wasm);
-    let clang_args = options.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap();
+
+    // This is a default for backwards-compatibility with the original
+    // definition of this target oh-so-long-ago. Once the "wasm" ABI is
+    // stable and the wasm-bindgen project has switched to using it then there's
+    // no need for this and it can be removed.
+    //
+    // Currently this is the reason that this target's ABI is mismatched with
+    // clang's ABI. This means that, in the limit, you can't merge C and Rust
+    // code on this target due to this ABI mismatch.
+    options.default_adjusted_cabi = Some(Abi::Wasm);
+
+    let clang_args = options.pre_link_args.entry(LinkerFlavor::Gcc).or_default();
 
     // Make sure clang uses LLD as its linker and is configured appropriately
     // otherwise
@@ -35,7 +47,7 @@ pub fn target() -> Target {
     clang_args.push("-Wl,--export-dynamic".to_string());
 
     // Add the flags to wasm-ld's args too.
-    let lld_args = options.pre_link_args.get_mut(&LinkerFlavor::Lld(LldFlavor::Wasm)).unwrap();
+    let lld_args = options.pre_link_args.entry(LinkerFlavor::Lld(LldFlavor::Wasm)).or_default();
     lld_args.push("--no-entry".to_string());
     lld_args.push("--export-dynamic".to_string());
 
diff --git a/compiler/rustc_target/src/spec/wasm32_wasi.rs b/compiler/rustc_target/src/spec/wasm32_wasi.rs
index 3f44acdc36b..a6b12d2ee8f 100644
--- a/compiler/rustc_target/src/spec/wasm32_wasi.rs
+++ b/compiler/rustc_target/src/spec/wasm32_wasi.rs
@@ -72,11 +72,11 @@
 //! best we can with this target. Don't start relying on too much here unless
 //! you know what you're getting in to!
 
-use super::wasm32_base;
+use super::wasm_base;
 use super::{crt_objects, LinkerFlavor, LldFlavor, Target};
 
 pub fn target() -> Target {
-    let mut options = wasm32_base::options();
+    let mut options = wasm_base::options();
 
     options.os = "wasi".to_string();
     options.linker_flavor = LinkerFlavor::Lld(LldFlavor::Wasm);
diff --git a/compiler/rustc_target/src/spec/wasm64_unknown_unknown.rs b/compiler/rustc_target/src/spec/wasm64_unknown_unknown.rs
new file mode 100644
index 00000000000..8bfb229d77f
--- /dev/null
+++ b/compiler/rustc_target/src/spec/wasm64_unknown_unknown.rs
@@ -0,0 +1,39 @@
+//! A "bare wasm" target representing a WebAssembly output that makes zero
+//! assumptions about its environment.
+//!
+//! The `wasm64-unknown-unknown` target is intended to encapsulate use cases
+//! that do not rely on any imported functionality. The binaries generated are
+//! entirely self-contained by default when using the standard library. Although
+//! the standard library is available, most of it returns an error immediately
+//! (e.g. trying to create a TCP stream or something like that).
+
+use super::wasm_base;
+use super::{LinkerFlavor, LldFlavor, Target};
+
+pub fn target() -> Target {
+    let mut options = wasm_base::options();
+    options.os = "unknown".to_string();
+    options.linker_flavor = LinkerFlavor::Lld(LldFlavor::Wasm);
+    let clang_args = options.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap();
+
+    // Make sure clang uses LLD as its linker and is configured appropriately
+    // otherwise
+    clang_args.push("--target=wasm64-unknown-unknown".to_string());
+
+    // For now this target just never has an entry symbol no matter the output
+    // type, so unconditionally pass this.
+    clang_args.push("-Wl,--no-entry".to_string());
+    options
+        .pre_link_args
+        .get_mut(&LinkerFlavor::Lld(LldFlavor::Wasm))
+        .unwrap()
+        .push("--no-entry".to_string());
+
+    Target {
+        llvm_target: "wasm64-unknown-unknown".to_string(),
+        pointer_width: 64,
+        data_layout: "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(),
+        arch: "wasm64".to_string(),
+        options,
+    }
+}
diff --git a/compiler/rustc_target/src/spec/wasm32_base.rs b/compiler/rustc_target/src/spec/wasm_base.rs
index bfef3d37228..b208eb92f8f 100644
--- a/compiler/rustc_target/src/spec/wasm32_base.rs
+++ b/compiler/rustc_target/src/spec/wasm_base.rs
@@ -60,6 +60,8 @@ pub fn options() -> TargetOptions {
     pre_link_args.insert(LinkerFlavor::Gcc, clang_args);
 
     TargetOptions {
+        is_like_wasm: true,
+
         // we allow dynamic linking, but only cdylibs. Basically we allow a
         // final library artifact that exports some symbols (a wasm module) but
         // we don't allow intermediate `dylib` crate types
@@ -73,7 +75,6 @@ pub fn options() -> TargetOptions {
         exe_suffix: ".wasm".to_string(),
         dll_prefix: String::new(),
         dll_suffix: ".wasm".to_string(),
-        linker_is_gnu: false,
         eh_frame_header: false,
 
         max_atomic_width: Some(64),
diff --git a/compiler/rustc_target/src/spec/windows_gnu_base.rs b/compiler/rustc_target/src/spec/windows_gnu_base.rs
index f556a13a519..478c567a93b 100644
--- a/compiler/rustc_target/src/spec/windows_gnu_base.rs
+++ b/compiler/rustc_target/src/spec/windows_gnu_base.rs
@@ -9,8 +9,6 @@ pub fn opts() -> TargetOptions {
             // Tell GCC to avoid linker plugins, because we are not bundling
             // them with Windows installer, and Rust does its own LTO anyways.
             "-fno-use-linker-plugin".to_string(),
-            // Always enable DEP (NX bit) when it is available
-            "-Wl,--nxcompat".to_string(),
             // Enable ASLR
             "-Wl,--dynamicbase".to_string(),
             // ASLR will rebase it anyway so leaving that option enabled only leads to confusion
@@ -73,8 +71,6 @@ pub fn opts() -> TargetOptions {
         dll_prefix: String::new(),
         dll_suffix: ".dll".to_string(),
         exe_suffix: ".exe".to_string(),
-        staticlib_prefix: "lib".to_string(),
-        staticlib_suffix: ".a".to_string(),
         os_family: Some("windows".to_string()),
         is_like_windows: true,
         allows_weak_linkage: false,
diff --git a/compiler/rustc_target/src/spec/windows_uwp_msvc_base.rs b/compiler/rustc_target/src/spec/windows_uwp_msvc_base.rs
index 700ee5ec646..b3fa5c22f98 100644
--- a/compiler/rustc_target/src/spec/windows_uwp_msvc_base.rs
+++ b/compiler/rustc_target/src/spec/windows_uwp_msvc_base.rs
@@ -5,10 +5,10 @@ pub fn opts() -> TargetOptions {
 
     opts.vendor = "uwp".to_string();
     let pre_link_args_msvc = vec!["/APPCONTAINER".to_string(), "mincore.lib".to_string()];
-    opts.pre_link_args.get_mut(&LinkerFlavor::Msvc).unwrap().extend(pre_link_args_msvc.clone());
+    opts.pre_link_args.entry(LinkerFlavor::Msvc).or_default().extend(pre_link_args_msvc.clone());
     opts.pre_link_args
-        .get_mut(&LinkerFlavor::Lld(LldFlavor::Link))
-        .unwrap()
+        .entry(LinkerFlavor::Lld(LldFlavor::Link))
+        .or_default()
         .extend(pre_link_args_msvc);
 
     opts
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs b/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs
index 8c40baccda8..c82359223da 100644
--- a/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs
@@ -1,4 +1,4 @@
-use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, SanitizerSet, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::apple_base::opts("macos");
@@ -11,6 +11,7 @@ pub fn target() -> Target {
     );
     base.link_env_remove.extend(super::apple_base::macos_link_env_remove());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::THREAD;
 
     // Clang automatically chooses a more specific target based on
     // MACOSX_DEPLOYMENT_TARGET.  To enable cross-language LTO to work
diff --git a/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs b/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs
index 74fb6f0a834..90705c526f4 100644
--- a/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs
+++ b/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs
@@ -1,12 +1,9 @@
 use std::iter;
 
-use super::{LinkerFlavor, LldFlavor, PanicStrategy, Target, TargetOptions};
+use super::{LinkerFlavor, LldFlavor, Target, TargetOptions};
 
 pub fn target() -> Target {
     const PRE_LINK_ARGS: &[&str] = &[
-        "--as-needed",
-        "-z",
-        "noexecstack",
         "-e",
         "elf_entry",
         "-Bstatic",
@@ -59,12 +56,10 @@ pub fn target() -> Target {
         env: "sgx".into(),
         vendor: "fortanix".into(),
         linker_flavor: LinkerFlavor::Lld(LldFlavor::Ld),
-        dynamic_linking: false,
         executables: true,
         linker_is_gnu: true,
         linker: Some("rust-lld".to_owned()),
         max_atomic_width: Some(64),
-        panic_strategy: PanicStrategy::Unwind,
         cpu: "x86-64".into(),
         features: "+rdrnd,+rdseed,+lvi-cfi,+lvi-load-hardening".into(),
         llvm_args: vec!["--x86-experimental-lvi-inline-asm-hardening".into()],
diff --git a/compiler/rustc_target/src/spec/x86_64_fuchsia.rs b/compiler/rustc_target/src/spec/x86_64_fuchsia.rs
index a39e7f8c341..99acc7c207b 100644
--- a/compiler/rustc_target/src/spec/x86_64_fuchsia.rs
+++ b/compiler/rustc_target/src/spec/x86_64_fuchsia.rs
@@ -1,10 +1,11 @@
-use crate::spec::{StackProbeType, Target};
+use crate::spec::{SanitizerSet, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::fuchsia_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
+    base.supported_sanitizers = SanitizerSet::ADDRESS;
 
     Target {
         llvm_target: "x86_64-fuchsia".to_string(),
diff --git a/compiler/rustc_target/src/spec/x86_64_linux_android.rs b/compiler/rustc_target/src/spec/x86_64_linux_android.rs
index d436242e62b..0945a9f5c59 100644
--- a/compiler/rustc_target/src/spec/x86_64_linux_android.rs
+++ b/compiler/rustc_target/src/spec/x86_64_linux_android.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     // https://developer.android.com/ndk/guides/abis.html#86-64
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/x86_64_pc_windows_gnu.rs b/compiler/rustc_target/src/spec/x86_64_pc_windows_gnu.rs
index 36726ab4aed..26a81a484b9 100644
--- a/compiler/rustc_target/src/spec/x86_64_pc_windows_gnu.rs
+++ b/compiler/rustc_target/src/spec/x86_64_pc_windows_gnu.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, LldFlavor, Target};
 pub fn target() -> Target {
     let mut base = super::windows_gnu_base::opts();
     base.cpu = "x86-64".to_string();
-    let gcc_pre_link_args = base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap();
+    let gcc_pre_link_args = base.pre_link_args.entry(LinkerFlavor::Gcc).or_default();
     gcc_pre_link_args.push("-m64".to_string());
     // Use high-entropy 64 bit address space for ASLR
     gcc_pre_link_args.push("-Wl,--high-entropy-va".to_string());
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_dragonfly.rs b/compiler/rustc_target/src/spec/x86_64_unknown_dragonfly.rs
index d86b0d67acd..295f9c837c3 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_dragonfly.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_dragonfly.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::dragonfly_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs
index c7d3b3feed5..ca3556fc48e 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs
@@ -1,11 +1,12 @@
-use crate::spec::{LinkerFlavor, StackProbeType, Target};
+use crate::spec::{LinkerFlavor, SanitizerSet, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::MEMORY | SanitizerSet::THREAD;
 
     Target {
         llvm_target: "x86_64-unknown-freebsd".to_string(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs
index 99906764dfc..9569e98ed59 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs
@@ -1,11 +1,13 @@
-use crate::spec::{LinkerFlavor, StackProbeType, Target};
+use crate::spec::{LinkerFlavor, SanitizerSet, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
+    base.supported_sanitizers =
+        SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::MEMORY | SanitizerSet::THREAD;
 
     Target {
         llvm_target: "x86_64-unknown-linux-gnu".to_string(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnux32.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnux32.rs
index 4b2bce37470..5f87534fe95 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnux32.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnux32.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-mx32".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-mx32".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
     base.has_elf_tls = false;
     // BUG(GabrielMajeri): disabling the PLT on x86_64 Linux with x32 ABI
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs
index fa9fdf5aa09..1e2e5766a31 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs
@@ -1,12 +1,14 @@
-use crate::spec::{LinkerFlavor, StackProbeType, Target};
+use crate::spec::{LinkerFlavor, SanitizerSet, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
     base.static_position_independent_executables = true;
+    base.supported_sanitizers =
+        SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::MEMORY | SanitizerSet::THREAD;
 
     Target {
         llvm_target: "x86_64-unknown-linux-musl".to_string(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs
index 6d19dec00b4..54e7ceee82e 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_hermit_kernel.rs b/compiler/rustc_target/src/spec/x86_64_unknown_none_hermitkernel.rs
index 9fcfe4e6c14..a357def190b 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_hermit_kernel.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_none_hermitkernel.rs
@@ -10,7 +10,7 @@ pub fn target() -> Target {
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
-        llvm_target: "x86_64-unknown-hermit".to_string(),
+        llvm_target: "x86_64-unknown-none-elf".to_string(),
         pointer_width: 64,
         data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
             .to_string(),
diff --git a/compiler/rustc_target/src/spec/x86_64_linux_kernel.rs b/compiler/rustc_target/src/spec/x86_64_unknown_none_linuxkernel.rs
index 43e683ddbcc..fa6f255d4d9 100644
--- a/compiler/rustc_target/src/spec/x86_64_linux_kernel.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_none_linuxkernel.rs
@@ -11,11 +11,14 @@ pub fn target() -> Target {
         "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-3dnow,-3dnowa,-avx,-avx2,+soft-float"
             .to_string();
     base.code_model = Some(CodeModel::Kernel);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
 
     Target {
-        // FIXME: Some dispute, the linux-on-clang folks think this should use "Linux"
-        llvm_target: "x86_64-elf".to_string(),
+        // FIXME: Some dispute, the linux-on-clang folks think this should use
+        // "Linux". We disagree because running *on* Linux is nothing like
+        // running *as" linux, and historically the "os" component as has always
+        // been used to mean the "on" part.
+        llvm_target: "x86_64-unknown-none-elf".to_string(),
         pointer_width: 64,
         data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
             .to_string(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs
index ac5939bcb3c..530e63966aa 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::openbsd_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_redox.rs b/compiler/rustc_target/src/spec/x86_64_unknown_redox.rs
index ddabe95ab83..934f8de8ecc 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_redox.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_redox.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::redox_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
 
     Target {
diff --git a/compiler/rustc_target/src/spec/x86_64_uwp_windows_gnu.rs b/compiler/rustc_target/src/spec/x86_64_uwp_windows_gnu.rs
index 57913ba0dab..a5425e1c129 100644
--- a/compiler/rustc_target/src/spec/x86_64_uwp_windows_gnu.rs
+++ b/compiler/rustc_target/src/spec/x86_64_uwp_windows_gnu.rs
@@ -3,7 +3,7 @@ use crate::spec::{LinkerFlavor, LldFlavor, Target};
 pub fn target() -> Target {
     let mut base = super::windows_uwp_gnu_base::opts();
     base.cpu = "x86-64".to_string();
-    let gcc_pre_link_args = base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap();
+    let gcc_pre_link_args = base.pre_link_args.entry(LinkerFlavor::Gcc).or_default();
     gcc_pre_link_args.push("-m64".to_string());
     // Use high-entropy 64 bit address space for ASLR
     gcc_pre_link_args.push("-Wl,--high-entropy-va".to_string());
diff --git a/compiler/rustc_target/src/spec/x86_64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/x86_64_wrs_vxworks.rs
index 1b35e813fcd..f9fa9d93843 100644
--- a/compiler/rustc_target/src/spec/x86_64_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/x86_64_wrs_vxworks.rs
@@ -4,7 +4,7 @@ pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.pre_link_args.entry(LinkerFlavor::Gcc).or_default().push("-m64".to_string());
     base.stack_probes = StackProbeType::InlineOrCall { min_llvm_version_for_inline: (11, 0, 1) };
     base.disable_redzone = true;
 
diff --git a/compiler/rustc_trait_selection/src/infer.rs b/compiler/rustc_trait_selection/src/infer.rs
index da66fbc8587..a9ffb5542b6 100644
--- a/compiler/rustc_trait_selection/src/infer.rs
+++ b/compiler/rustc_trait_selection/src/infer.rs
@@ -124,7 +124,7 @@ impl<'tcx> InferCtxtBuilderExt<'tcx> for InferCtxtBuilder<'tcx> {
             DUMMY_SP,
             canonical_key,
             |ref infcx, key, canonical_inference_vars| {
-                let mut fulfill_cx = TraitEngine::new(infcx.tcx);
+                let mut fulfill_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
                 let value = operation(infcx, &mut *fulfill_cx, key)?;
                 infcx.make_canonicalized_query_response(
                     canonical_inference_vars,
diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs
index e1f8d59991f..4097e1577e1 100644
--- a/compiler/rustc_trait_selection/src/lib.rs
+++ b/compiler/rustc_trait_selection/src/lib.rs
@@ -15,15 +15,16 @@
 #![feature(box_patterns)]
 #![feature(drain_filter)]
 #![feature(in_band_lifetimes)]
+#![feature(iter_zip)]
 #![feature(never_type)]
 #![feature(crate_visibility_modifier)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(control_flow_enum)]
 #![recursion_limit = "512"] // For rustdoc
 
 #[macro_use]
 extern crate rustc_macros;
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 #[macro_use]
 extern crate rustc_data_structures;
 #[macro_use]
diff --git a/compiler/rustc_trait_selection/src/opaque_types.rs b/compiler/rustc_trait_selection/src/opaque_types.rs
index 25ba489032b..fb4a8ce687c 100644
--- a/compiler/rustc_trait_selection/src/opaque_types.rs
+++ b/compiler/rustc_trait_selection/src/opaque_types.rs
@@ -422,7 +422,9 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
             }
             // These opaque type inherit all lifetime parameters from their
             // parent, so we have to check them all.
-            hir::OpaqueTyOrigin::Binding | hir::OpaqueTyOrigin::Misc => 0,
+            hir::OpaqueTyOrigin::Binding
+            | hir::OpaqueTyOrigin::TyAlias
+            | hir::OpaqueTyOrigin::Misc => 0,
         };
 
         let span = tcx.def_span(def_id);
@@ -581,6 +583,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
             // Otherwise, generate the label we'll use in the error message.
             hir::OpaqueTyOrigin::Binding
             | hir::OpaqueTyOrigin::FnReturn
+            | hir::OpaqueTyOrigin::TyAlias
             | hir::OpaqueTyOrigin::Misc => "impl Trait",
         };
         let msg = format!("ambiguous lifetime bound in `{}`", context_name);
@@ -694,7 +697,7 @@ where
 {
     fn visit_binder<T: TypeFoldable<'tcx>>(
         &mut self,
-        t: &ty::Binder<T>,
+        t: &ty::Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
         t.as_ref().skip_binder().visit_with(self);
         ControlFlow::CONTINUE
@@ -1168,7 +1171,7 @@ impl<'a, 'tcx> Instantiator<'a, 'tcx> {
             // This also instantiates nested instances of `impl Trait`.
             let predicate = self.instantiate_opaque_types_in_map(predicate);
 
-            let cause = traits::ObligationCause::new(span, self.body_id, traits::MiscObligation);
+            let cause = traits::ObligationCause::new(span, self.body_id, traits::OpaqueType);
 
             // Require that the predicate holds for the concrete type.
             debug!("instantiate_opaque_types: predicate={:?}", predicate);
diff --git a/compiler/rustc_trait_selection/src/traits/auto_trait.rs b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
index 6510c9464e1..f54eb0914a5 100644
--- a/compiler/rustc_trait_selection/src/traits/auto_trait.rs
+++ b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
@@ -12,6 +12,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 
 use std::collections::hash_map::Entry;
 use std::collections::VecDeque;
+use std::iter;
 
 // FIXME(twk): this is obviously not nice to duplicate like that
 #[derive(Eq, PartialEq, Hash, Copy, Clone, Debug)]
@@ -83,7 +84,7 @@ impl<'tcx> AutoTraitFinder<'tcx> {
 
         let trait_ref = ty::TraitRef { def_id: trait_did, substs: tcx.mk_substs_trait(ty, &[]) };
 
-        let trait_pred = ty::Binder::bind(trait_ref);
+        let trait_pred = ty::Binder::dummy(trait_ref);
 
         let bail_out = tcx.infer_ctxt().enter(|infcx| {
             let mut selcx = SelectionContext::with_negative(&infcx, true);
@@ -279,7 +280,7 @@ impl AutoTraitFinder<'tcx> {
 
         let mut already_visited = FxHashSet::default();
         let mut predicates = VecDeque::new();
-        predicates.push_back(ty::Binder::bind(ty::TraitPredicate {
+        predicates.push_back(ty::Binder::dummy(ty::TraitPredicate {
             trait_ref: ty::TraitRef {
                 def_id: trait_did,
                 substs: infcx.tcx.mk_substs_trait(ty, &[]),
@@ -428,7 +429,9 @@ impl AutoTraitFinder<'tcx> {
                         return true;
                     }
 
-                    for (new_region, old_region) in new_substs.regions().zip(old_substs.regions()) {
+                    for (new_region, old_region) in
+                        iter::zip(new_substs.regions(), old_substs.regions())
+                    {
                         match (new_region, old_region) {
                             // If both predicates have an `ReLateBound` (a HRTB) in the
                             // same spot, we do nothing.
@@ -803,12 +806,10 @@ impl AutoTraitFinder<'tcx> {
                 }
                 ty::PredicateKind::ConstEquate(c1, c2) => {
                     let evaluate = |c: &'tcx ty::Const<'tcx>| {
-                        if let ty::ConstKind::Unevaluated(def, substs, promoted) = c.val {
+                        if let ty::ConstKind::Unevaluated(unevaluated) = c.val {
                             match select.infcx().const_eval_resolve(
                                 obligation.param_env,
-                                def,
-                                substs,
-                                promoted,
+                                unevaluated,
                                 Some(obligation.cause.span),
                             ) {
                                 Ok(val) => Ok(ty::Const::from_value(select.tcx(), val, c.ty)),
diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs
index e8ae1f44a36..9bb4af16a8f 100644
--- a/compiler/rustc_trait_selection/src/traits/coherence.rs
+++ b/compiler/rustc_trait_selection/src/traits/coherence.rs
@@ -74,23 +74,22 @@ where
     let impl2_ref = tcx.impl_trait_ref(impl2_def_id);
 
     // Check if any of the input types definitely do not unify.
-    if impl1_ref
-        .iter()
-        .flat_map(|tref| tref.substs.types())
-        .zip(impl2_ref.iter().flat_map(|tref| tref.substs.types()))
-        .any(|(ty1, ty2)| {
-            let t1 = fast_reject::simplify_type(tcx, ty1, false);
-            let t2 = fast_reject::simplify_type(tcx, ty2, false);
-            if let (Some(t1), Some(t2)) = (t1, t2) {
-                // Simplified successfully
-                // Types cannot unify if they differ in their reference mutability or simplify to different types
-                t1 != t2 || ty1.ref_mutability() != ty2.ref_mutability()
-            } else {
-                // Types might unify
-                false
-            }
-        })
-    {
+    if iter::zip(
+        impl1_ref.iter().flat_map(|tref| tref.substs.types()),
+        impl2_ref.iter().flat_map(|tref| tref.substs.types()),
+    )
+    .any(|(ty1, ty2)| {
+        let t1 = fast_reject::simplify_type(tcx, ty1, false);
+        let t2 = fast_reject::simplify_type(tcx, ty2, false);
+        if let (Some(t1), Some(t2)) = (t1, t2) {
+            // Simplified successfully
+            // Types cannot unify if they differ in their reference mutability or simplify to different types
+            t1 != t2 || ty1.ref_mutability() != ty2.ref_mutability()
+        } else {
+            // Types might unify
+            false
+        }
+    }) {
         // Some types involved are definitely different, so the impls couldn't possibly overlap.
         debug!("overlapping_impls: fast_reject early-exit");
         return no_overlap();
@@ -587,6 +586,11 @@ fn ty_is_local_constructor(ty: Ty<'_>, in_crate: InCrate) -> bool {
             false
         }
 
+        ty::Closure(..) => {
+            // Similar to the `Opaque` case (#83613).
+            false
+        }
+
         ty::Dynamic(ref tt, ..) => {
             if let Some(principal) = tt.principal() {
                 def_id_is_local(principal.def_id(), in_crate)
@@ -597,7 +601,7 @@ fn ty_is_local_constructor(ty: Ty<'_>, in_crate: InCrate) -> bool {
 
         ty::Error(_) => true,
 
-        ty::Closure(..) | ty::Generator(..) | ty::GeneratorWitness(..) => {
+        ty::Generator(..) | ty::GeneratorWitness(..) => {
             bug!("ty_is_local invoked on unexpected type: {:?}", ty)
         }
     }
diff --git a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
index 8a1be7ea172..8961cdaebf3 100644
--- a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
+++ b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
@@ -13,7 +13,7 @@ use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
 use rustc_index::vec::IndexVec;
 use rustc_infer::infer::InferCtxt;
-use rustc_middle::mir::abstract_const::{Node, NodeId};
+use rustc_middle::mir::abstract_const::{Node, NodeId, NotConstEvaluatable};
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::mir::{self, Rvalue, StatementKind, TerminatorKind};
 use rustc_middle::ty::subst::{Subst, SubstsRef};
@@ -23,6 +23,7 @@ use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::Span;
 
 use std::cmp;
+use std::iter;
 use std::ops::ControlFlow;
 
 /// Check if a given constant can be evaluated.
@@ -32,7 +33,7 @@ pub fn is_const_evaluatable<'cx, 'tcx>(
     substs: SubstsRef<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     span: Span,
-) -> Result<(), ErrorHandled> {
+) -> Result<(), NotConstEvaluatable> {
     debug!("is_const_evaluatable({:?}, {:?})", def, substs);
     if infcx.tcx.features().const_evaluatable_checked {
         let tcx = infcx.tcx;
@@ -103,29 +104,10 @@ pub fn is_const_evaluatable<'cx, 'tcx>(
 
                 match failure_kind {
                     FailureKind::MentionsInfer => {
-                        return Err(ErrorHandled::TooGeneric);
+                        return Err(NotConstEvaluatable::MentionsInfer);
                     }
                     FailureKind::MentionsParam => {
-                        // FIXME(const_evaluatable_checked): Better error message.
-                        let mut err =
-                            infcx.tcx.sess.struct_span_err(span, "unconstrained generic constant");
-                        let const_span = tcx.def_span(def.did);
-                        // FIXME(const_evaluatable_checked): Update this suggestion once
-                        // explicit const evaluatable bounds are implemented.
-                        if let Ok(snippet) = infcx.tcx.sess.source_map().span_to_snippet(const_span)
-                        {
-                            err.span_help(
-                                tcx.def_span(def.did),
-                                &format!("try adding a `where` bound using this expression: `where [u8; {}]: Sized`", snippet),
-                            );
-                        } else {
-                            err.span_help(
-                                const_span,
-                                "consider adding a `where` bound for this expression",
-                            );
-                        }
-                        err.emit();
-                        return Err(ErrorHandled::Reported(ErrorReported));
+                        return Err(NotConstEvaluatable::MentionsParam);
                     }
                     FailureKind::Concrete => {
                         // Dealt with below by the same code which handles this
@@ -163,7 +145,11 @@ pub fn is_const_evaluatable<'cx, 'tcx>(
     // and hopefully soon change this to an error.
     //
     // See #74595 for more details about this.
-    let concrete = infcx.const_eval_resolve(param_env, def, substs, None, Some(span));
+    let concrete = infcx.const_eval_resolve(
+        param_env,
+        ty::Unevaluated { def, substs, promoted: None },
+        Some(span),
+    );
 
     if concrete.is_ok() && substs.has_param_types_or_consts() {
         match infcx.tcx.def_kind(def.did) {
@@ -180,34 +166,16 @@ pub fn is_const_evaluatable<'cx, 'tcx>(
 
     debug!(?concrete, "is_const_evaluatable");
     match concrete {
-        Err(ErrorHandled::TooGeneric) if !substs.has_infer_types_or_consts() => {
-            // FIXME(const_evaluatable_checked): We really should move
-            // emitting this error message to fulfill instead. For
-            // now this is easier.
-            //
-            // This is not a problem without `const_evaluatable_checked` as
-            // all `ConstEvaluatable` predicates have to be fulfilled for compilation
-            // to succeed.
-            //
-            // @lcnr: We already emit an error for things like
-            // `fn test<const N: usize>() -> [0 - N]` eagerly here,
-            // so until we fix this I don't really care.
-
-            let mut err = infcx
-                .tcx
-                .sess
-                .struct_span_err(span, "constant expression depends on a generic parameter");
-            // FIXME(const_generics): we should suggest to the user how they can resolve this
-            // issue. However, this is currently not actually possible
-            // (see https://github.com/rust-lang/rust/issues/66962#issuecomment-575907083).
-            //
-            // Note that with `feature(const_evaluatable_checked)` this case should not
-            // be reachable.
-            err.note("this may fail depending on what value the parameter takes");
-            err.emit();
-            Err(ErrorHandled::Reported(ErrorReported))
+        Err(ErrorHandled::TooGeneric) => Err(match substs.has_infer_types_or_consts() {
+            true => NotConstEvaluatable::MentionsInfer,
+            false => NotConstEvaluatable::MentionsParam,
+        }),
+        Err(ErrorHandled::Linted) => {
+            infcx.tcx.sess.delay_span_bug(span, "constant in type had error reported as lint");
+            Err(NotConstEvaluatable::Error(ErrorReported))
         }
-        c => c.map(drop),
+        Err(ErrorHandled::Reported(e)) => Err(NotConstEvaluatable::Error(e)),
+        Ok(_) => Ok(()),
     }
 }
 
@@ -239,7 +207,9 @@ impl AbstractConst<'tcx> {
         ct: &ty::Const<'tcx>,
     ) -> Result<Option<AbstractConst<'tcx>>, ErrorReported> {
         match ct.val {
-            ty::ConstKind::Unevaluated(def, substs, None) => AbstractConst::new(tcx, def, substs),
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted: _ }) => {
+                AbstractConst::new(tcx, def, substs)
+            }
             ty::ConstKind::Error(_) => Err(ErrorReported),
             _ => Ok(None),
         }
@@ -377,7 +347,10 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> {
                 let local = self.place_to_local(span, p)?;
                 Ok(self.locals[local])
             }
-            mir::Operand::Constant(ct) => Ok(self.add_node(Node::Leaf(ct.literal), span)),
+            mir::Operand::Constant(ct) => match ct.literal {
+                mir::ConstantKind::Ty(ct) => Ok(self.add_node(Node::Leaf(ct), span)),
+                mir::ConstantKind::Val(..) => self.error(Some(span), "unsupported constant")?,
+            },
         }
     }
 
@@ -411,7 +384,7 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> {
                         self.locals[local] = self.operand_to_node(span, operand)?;
                         Ok(())
                     }
-                    Rvalue::BinaryOp(op, ref lhs, ref rhs) if Self::check_binop(op) => {
+                    Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) if Self::check_binop(op) => {
                         let lhs = self.operand_to_node(span, lhs)?;
                         let rhs = self.operand_to_node(span, rhs)?;
                         self.locals[local] = self.add_node(Node::Binop(op, lhs, rhs), span);
@@ -421,7 +394,9 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> {
                             Ok(())
                         }
                     }
-                    Rvalue::CheckedBinaryOp(op, ref lhs, ref rhs) if Self::check_binop(op) => {
+                    Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs))
+                        if Self::check_binop(op) =>
+                    {
                         let lhs = self.operand_to_node(span, lhs)?;
                         let rhs = self.operand_to_node(span, rhs)?;
                         self.locals[local] = self.add_node(Node::Binop(op, lhs, rhs), span);
@@ -527,22 +502,25 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> {
             if let Some(next) = self.build_terminator(block.terminator())? {
                 block = &self.body.basic_blocks()[next];
             } else {
-                assert_eq!(self.locals[mir::RETURN_PLACE], self.nodes.last().unwrap());
+                break;
+            }
+        }
+
+        assert_eq!(self.locals[mir::RETURN_PLACE], self.nodes.last().unwrap());
+        for n in self.nodes.iter() {
+            if let Node::Leaf(ty::Const { val: ty::ConstKind::Unevaluated(ct), ty: _ }) = n.node {
                 // `AbstractConst`s should not contain any promoteds as they require references which
                 // are not allowed.
-                assert!(!self.nodes.iter().any(|n| matches!(
-                    n.node,
-                    Node::Leaf(ty::Const { val: ty::ConstKind::Unevaluated(_, _, Some(_)), ty: _ })
-                )));
-
-                self.nodes[self.locals[mir::RETURN_PLACE]].used = true;
-                if let Some(&unused) = self.nodes.iter().find(|n| !n.used) {
-                    self.error(Some(unused.span), "dead code")?;
-                }
-
-                return Ok(self.tcx.arena.alloc_from_iter(self.nodes.into_iter().map(|n| n.node)));
+                assert_eq!(ct.promoted, None);
             }
         }
+
+        self.nodes[self.locals[mir::RETURN_PLACE]].used = true;
+        if let Some(&unused) = self.nodes.iter().find(|n| !n.used) {
+            self.error(Some(unused.span), "dead code")?;
+        }
+
+        Ok(self.tcx.arena.alloc_from_iter(self.nodes.into_iter().map(|n| n.node)))
     }
 }
 
@@ -668,10 +646,16 @@ pub(super) fn try_unify<'tcx>(
                 // we do not want to use `assert_eq!(a(), b())` to infer that `N` and `M` have to be `1`. This
                 // means that we only allow inference variables if they are equal.
                 (ty::ConstKind::Infer(a_val), ty::ConstKind::Infer(b_val)) => a_val == b_val,
-                (
-                    ty::ConstKind::Unevaluated(a_def, a_substs, None),
-                    ty::ConstKind::Unevaluated(b_def, b_substs, None),
-                ) => a_def == b_def && a_substs == b_substs,
+                // We expand generic anonymous constants at the start of this function, so this
+                // branch should only be taking when dealing with associated constants, at
+                // which point directly comparing them seems like the desired behavior.
+                //
+                // FIXME(const_evaluatable_checked): This isn't actually the case.
+                // We also take this branch for concrete anonymous constants and
+                // expand generic anonymous constants with concrete substs.
+                (ty::ConstKind::Unevaluated(a_uv), ty::ConstKind::Unevaluated(b_uv)) => {
+                    a_uv == b_uv
+                }
                 // FIXME(const_evaluatable_checked): We may want to either actually try
                 // to evaluate `a_ct` and `b_ct` if they are are fully concrete or something like
                 // this, for now we just return false here.
@@ -689,9 +673,7 @@ pub(super) fn try_unify<'tcx>(
             if a_args.len() == b_args.len() =>
         {
             try_unify(tcx, a.subtree(a_f), b.subtree(b_f))
-                && a_args
-                    .iter()
-                    .zip(b_args)
+                && iter::zip(a_args, b_args)
                     .all(|(&an, &bn)| try_unify(tcx, a.subtree(an), b.subtree(bn)))
         }
         _ => false,
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
index bfb5ebcea58..da5a1af7f77 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
@@ -2,10 +2,10 @@ pub mod on_unimplemented;
 pub mod suggestions;
 
 use super::{
-    ConstEvalFailure, EvaluationResult, FulfillmentError, FulfillmentErrorCode,
-    MismatchedProjectionTypes, Obligation, ObligationCause, ObligationCauseCode,
-    OnUnimplementedDirective, OnUnimplementedNote, OutputTypeParameterMismatch, Overflow,
-    PredicateObligation, SelectionContext, SelectionError, TraitNotObjectSafe,
+    EvaluationResult, FulfillmentError, FulfillmentErrorCode, MismatchedProjectionTypes,
+    Obligation, ObligationCause, ObligationCauseCode, OnUnimplementedDirective,
+    OnUnimplementedNote, OutputTypeParameterMismatch, Overflow, PredicateObligation,
+    SelectionContext, SelectionError, TraitNotObjectSafe,
 };
 
 use crate::infer::error_reporting::{TyCategory, TypeAnnotationNeeded as ErrorCode};
@@ -17,7 +17,7 @@ use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::Node;
-use rustc_middle::mir::interpret::ErrorHandled;
+use rustc_middle::mir::abstract_const::NotConstEvaluatable;
 use rustc_middle::ty::error::ExpectedFound;
 use rustc_middle::ty::fold::TypeFolder;
 use rustc_middle::ty::{
@@ -28,6 +28,7 @@ use rustc_session::DiagnosticMessageId;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::{ExpnKind, MultiSpan, Span, DUMMY_SP};
 use std::fmt;
+use std::iter;
 
 use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
 use crate::traits::query::normalize::AtExt as _;
@@ -161,7 +162,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
             }
         }
 
-        for (error, suppressed) in errors.iter().zip(is_suppressed) {
+        for (error, suppressed) in iter::zip(errors, is_suppressed) {
             if !suppressed {
                 self.report_fulfillment_error(error, body_id, fallback_has_occurred);
             }
@@ -502,14 +503,14 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
                             let unit_obligation =
                                 obligation.with(predicate.without_const().to_predicate(tcx));
                             if self.predicate_may_hold(&unit_obligation) {
+                                err.note("this trait is implemented for `()`.");
                                 err.note(
-                                    "the trait is implemented for `()`. \
-                                     Possibly this error has been caused by changes to \
-                                     Rust's type-inference algorithm (see issue #48950 \
-                                     <https://github.com/rust-lang/rust/issues/48950> \
-                                     for more information). Consider whether you meant to use \
-                                     the type `()` here instead.",
+                                    "this error might have been caused by changes to \
+                                    Rust's type-inference algorithm (see issue #48950 \
+                                    <https://github.com/rust-lang/rust/issues/48950> \
+                                    for more information).",
                                 );
+                                err.help("did you intend to use the type `()` here instead?");
                             }
                         }
 
@@ -738,24 +739,59 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
                 let violations = self.tcx.object_safety_violations(did);
                 report_object_safety_error(self.tcx, span, did, violations)
             }
-            ConstEvalFailure(ErrorHandled::TooGeneric) => {
-                bug!("too generic should have been handled in `is_const_evaluatable`");
+
+            SelectionError::NotConstEvaluatable(NotConstEvaluatable::MentionsInfer) => {
+                bug!(
+                    "MentionsInfer should have been handled in `traits/fulfill.rs` or `traits/select/mod.rs`"
+                )
+            }
+            SelectionError::NotConstEvaluatable(NotConstEvaluatable::MentionsParam) => {
+                if !self.tcx.features().const_evaluatable_checked {
+                    let mut err = self.tcx.sess.struct_span_err(
+                        span,
+                        "constant expression depends on a generic parameter",
+                    );
+                    // FIXME(const_generics): we should suggest to the user how they can resolve this
+                    // issue. However, this is currently not actually possible
+                    // (see https://github.com/rust-lang/rust/issues/66962#issuecomment-575907083).
+                    //
+                    // Note that with `feature(const_evaluatable_checked)` this case should not
+                    // be reachable.
+                    err.note("this may fail depending on what value the parameter takes");
+                    err.emit();
+                    return;
+                }
+
+                match obligation.predicate.kind().skip_binder() {
+                    ty::PredicateKind::ConstEvaluatable(def, _) => {
+                        let mut err =
+                            self.tcx.sess.struct_span_err(span, "unconstrained generic constant");
+                        let const_span = self.tcx.def_span(def.did);
+                        match self.tcx.sess.source_map().span_to_snippet(const_span) {
+                            Ok(snippet) => err.help(&format!(
+                                "try adding a `where` bound using this expression: `where [(); {}]:`",
+                                snippet
+                            )),
+                            _ => err.help("consider adding a `where` bound using this expression"),
+                        };
+                        err
+                    }
+                    _ => {
+                        span_bug!(
+                            span,
+                            "unexpected non-ConstEvaluatable predicate, this should not be reachable"
+                        )
+                    }
+                }
             }
+
             // Already reported in the query.
-            ConstEvalFailure(ErrorHandled::Reported(ErrorReported)) => {
+            SelectionError::NotConstEvaluatable(NotConstEvaluatable::Error(ErrorReported)) => {
                 // FIXME(eddyb) remove this once `ErrorReported` becomes a proof token.
                 self.tcx.sess.delay_span_bug(span, "`ErrorReported` without an error");
                 return;
             }
 
-            // Already reported in the query, but only as a lint.
-            // This shouldn't actually happen for constants used in types, modulo
-            // bugs. The `delay_span_bug` here ensures it won't be ignored.
-            ConstEvalFailure(ErrorHandled::Linted) => {
-                self.tcx.sess.delay_span_bug(span, "constant in type had error reported as lint");
-                return;
-            }
-
             Overflow => {
                 bug!("overflow should be handled before the `report_selection_error` path");
             }
@@ -819,7 +855,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
                 sig.decl
                     .inputs
                     .iter()
-                    .map(|arg| match arg.clone().kind {
+                    .map(|arg| match arg.kind {
                         hir::TyKind::Tup(ref tys) => ArgKind::Tuple(
                             Some(arg.span),
                             vec![("_".to_owned(), "_".to_owned()); tys.len()],
@@ -1190,10 +1226,11 @@ impl<'a, 'tcx> InferCtxtPrivExt<'tcx> for InferCtxt<'a, 'tcx> {
                 );
 
                 let is_normalized_ty_expected = !matches!(
-                    obligation.cause.code,
+                    obligation.cause.code.peel_derives(),
                     ObligationCauseCode::ItemObligation(_)
                         | ObligationCauseCode::BindingObligation(_, _)
                         | ObligationCauseCode::ObjectCastObligation(_)
+                        | ObligationCauseCode::OpaqueType
                 );
 
                 if let Err(error) = self.at(&obligation.cause, obligation.param_env).eq_exp(
@@ -1917,7 +1954,7 @@ pub enum ArgKind {
     Arg(String, String),
 
     /// An argument of tuple type. For a "found" argument, the span is
-    /// the locationo in the source of the pattern. For a "expected"
+    /// the location in the source of the pattern. For a "expected"
     /// argument, it will be None. The vector is a list of (name, ty)
     /// strings for the components of the tuple.
     Tuple(Option<Span>, Vec<(String, String)>),
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
index e6a1cf58fe3..1ea34e5814e 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
@@ -7,6 +7,7 @@ use rustc_hir::def_id::DefId;
 use rustc_middle::ty::subst::Subst;
 use rustc_middle::ty::{self, GenericParamDefKind};
 use rustc_span::symbol::sym;
+use std::iter;
 
 use super::InferCtxtPrivExt;
 
@@ -51,12 +52,11 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
             if let Ok(..) = self.can_eq(param_env, trait_self_ty, impl_self_ty) {
                 self_match_impls.push(def_id);
 
-                if trait_ref
-                    .substs
-                    .types()
-                    .skip(1)
-                    .zip(impl_trait_ref.substs.types().skip(1))
-                    .all(|(u, v)| self.fuzzy_match_tys(u, v))
+                if iter::zip(
+                    trait_ref.substs.types().skip(1),
+                    impl_trait_ref.substs.types().skip(1),
+                )
+                .all(|(u, v)| self.fuzzy_match_tys(u, v))
                 {
                     fuzzy_match_impls.push(def_id);
                 }
@@ -163,61 +163,65 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
             flags.push((sym::from_desugaring, None));
             flags.push((sym::from_desugaring, Some(format!("{:?}", k))));
         }
-        let generics = self.tcx.generics_of(def_id);
-        let self_ty = trait_ref.self_ty();
-        // This is also included through the generics list as `Self`,
-        // but the parser won't allow you to use it
-        flags.push((sym::_Self, Some(self_ty.to_string())));
-        if let Some(def) = self_ty.ty_adt_def() {
-            // We also want to be able to select self's original
-            // signature with no type arguments resolved
-            flags.push((sym::_Self, Some(self.tcx.type_of(def.did).to_string())));
-        }
 
-        for param in generics.params.iter() {
-            let value = match param.kind {
-                GenericParamDefKind::Type { .. } | GenericParamDefKind::Const => {
-                    trait_ref.substs[param.index as usize].to_string()
-                }
-                GenericParamDefKind::Lifetime => continue,
-            };
-            let name = param.name;
-            flags.push((name, Some(value)));
-        }
+        // Add all types without trimmed paths.
+        ty::print::with_no_trimmed_paths(|| {
+            let generics = self.tcx.generics_of(def_id);
+            let self_ty = trait_ref.self_ty();
+            // This is also included through the generics list as `Self`,
+            // but the parser won't allow you to use it
+            flags.push((sym::_Self, Some(self_ty.to_string())));
+            if let Some(def) = self_ty.ty_adt_def() {
+                // We also want to be able to select self's original
+                // signature with no type arguments resolved
+                flags.push((sym::_Self, Some(self.tcx.type_of(def.did).to_string())));
+            }
 
-        if let Some(true) = self_ty.ty_adt_def().map(|def| def.did.is_local()) {
-            flags.push((sym::crate_local, None));
-        }
+            for param in generics.params.iter() {
+                let value = match param.kind {
+                    GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
+                        trait_ref.substs[param.index as usize].to_string()
+                    }
+                    GenericParamDefKind::Lifetime => continue,
+                };
+                let name = param.name;
+                flags.push((name, Some(value)));
+            }
 
-        // Allow targeting all integers using `{integral}`, even if the exact type was resolved
-        if self_ty.is_integral() {
-            flags.push((sym::_Self, Some("{integral}".to_owned())));
-        }
+            if let Some(true) = self_ty.ty_adt_def().map(|def| def.did.is_local()) {
+                flags.push((sym::crate_local, None));
+            }
 
-        if let ty::Array(aty, len) = self_ty.kind() {
-            flags.push((sym::_Self, Some("[]".to_owned())));
-            flags.push((sym::_Self, Some(format!("[{}]", aty))));
-            if let Some(def) = aty.ty_adt_def() {
-                // We also want to be able to select the array's type's original
-                // signature with no type arguments resolved
-                let type_string = self.tcx.type_of(def.did).to_string();
-                flags.push((sym::_Self, Some(format!("[{}]", type_string))));
+            // Allow targeting all integers using `{integral}`, even if the exact type was resolved
+            if self_ty.is_integral() {
+                flags.push((sym::_Self, Some("{integral}".to_owned())));
+            }
 
-                let len = len.val.try_to_value().and_then(|v| v.try_to_machine_usize(self.tcx));
-                let string = match len {
-                    Some(n) => format!("[{}; {}]", type_string, n),
-                    None => format!("[{}; _]", type_string),
-                };
-                flags.push((sym::_Self, Some(string)));
+            if let ty::Array(aty, len) = self_ty.kind() {
+                flags.push((sym::_Self, Some("[]".to_owned())));
+                flags.push((sym::_Self, Some(format!("[{}]", aty))));
+                if let Some(def) = aty.ty_adt_def() {
+                    // We also want to be able to select the array's type's original
+                    // signature with no type arguments resolved
+                    let type_string = self.tcx.type_of(def.did).to_string();
+                    flags.push((sym::_Self, Some(format!("[{}]", type_string))));
+
+                    let len = len.val.try_to_value().and_then(|v| v.try_to_machine_usize(self.tcx));
+                    let string = match len {
+                        Some(n) => format!("[{}; {}]", type_string, n),
+                        None => format!("[{}; _]", type_string),
+                    };
+                    flags.push((sym::_Self, Some(string)));
+                }
             }
-        }
-        if let ty::Dynamic(traits, _) = self_ty.kind() {
-            for t in traits.iter() {
-                if let ty::ExistentialPredicate::Trait(trait_ref) = t.skip_binder() {
-                    flags.push((sym::_Self, Some(self.tcx.def_path_str(trait_ref.def_id))))
+            if let ty::Dynamic(traits, _) = self_ty.kind() {
+                for t in traits.iter() {
+                    if let ty::ExistentialPredicate::Trait(trait_ref) = t.skip_binder() {
+                        flags.push((sym::_Self, Some(self.tcx.def_path_str(trait_ref.def_id))))
+                    }
                 }
             }
-        }
+        });
 
         if let Ok(Some(command)) =
             OnUnimplementedDirective::of_item(self.tcx, trait_ref.def_id, def_id)
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index 5c97791530d..6a4d41ffc1a 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -28,6 +28,7 @@ use std::fmt;
 
 use super::InferCtxtPrivExt;
 use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
+use rustc_middle::ty::print::with_no_trimmed_paths;
 
 #[derive(Debug)]
 pub enum GeneratorInteriorOrUpvar {
@@ -65,7 +66,7 @@ pub trait InferCtxtExt<'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         points_at_arg: bool,
     );
 
@@ -73,7 +74,7 @@ pub trait InferCtxtExt<'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: &ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: &ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         points_at_arg: bool,
         has_custom_message: bool,
     ) -> bool;
@@ -82,14 +83,14 @@ pub trait InferCtxtExt<'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     );
 
     fn suggest_change_mut(
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         points_at_arg: bool,
     );
 
@@ -98,7 +99,7 @@ pub trait InferCtxtExt<'tcx> {
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
         span: Span,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     );
 
     fn return_type_span(&self, obligation: &PredicateObligation<'tcx>) -> Option<Span>;
@@ -108,7 +109,7 @@ pub trait InferCtxtExt<'tcx> {
         err: &mut DiagnosticBuilder<'_>,
         span: Span,
         obligation: &PredicateObligation<'tcx>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     ) -> bool;
 
     fn point_at_returns_when_relevant(
@@ -170,7 +171,7 @@ pub trait InferCtxtExt<'tcx> {
         &self,
         err: &mut DiagnosticBuilder<'_>,
         obligation: &PredicateObligation<'tcx>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         span: Span,
     );
 }
@@ -440,7 +441,8 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
                 {
                     // Missing generic type parameter bound.
                     let param_name = self_ty.to_string();
-                    let constraint = trait_ref.print_only_trait_path().to_string();
+                    let constraint =
+                        with_no_trimmed_paths(|| trait_ref.print_only_trait_path().to_string());
                     if suggest_constraining_type_param(
                         self.tcx,
                         generics,
@@ -583,7 +585,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         points_at_arg: bool,
     ) {
         let self_ty = match trait_ref.self_ty().no_bound_vars() {
@@ -676,7 +678,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: &ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: &ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         points_at_arg: bool,
         has_custom_message: bool,
     ) -> bool {
@@ -761,7 +763,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     ) {
         let span = obligation.cause.span;
 
@@ -824,7 +826,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         points_at_arg: bool,
     ) {
         let span = obligation.cause.span;
@@ -896,10 +898,10 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
         obligation: &PredicateObligation<'tcx>,
         err: &mut DiagnosticBuilder<'_>,
         span: Span,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     ) {
         let is_empty_tuple =
-            |ty: ty::Binder<Ty<'_>>| *ty.skip_binder().kind() == ty::Tuple(ty::List::empty());
+            |ty: ty::Binder<'tcx, Ty<'_>>| *ty.skip_binder().kind() == ty::Tuple(ty::List::empty());
 
         let hir = self.tcx.hir();
         let parent_node = hir.get_parent_node(obligation.cause.body_id);
@@ -948,7 +950,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
         err: &mut DiagnosticBuilder<'_>,
         span: Span,
         obligation: &PredicateObligation<'tcx>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     ) -> bool {
         match obligation.cause.code.peel_derives() {
             // Only suggest `impl Trait` if the return type is unsized because it is `dyn Trait`.
@@ -1840,6 +1842,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
             | ObligationCauseCode::MethodReceiver
             | ObligationCauseCode::ReturnNoExpression
             | ObligationCauseCode::UnifyReceiver(..)
+            | ObligationCauseCode::OpaqueType
             | ObligationCauseCode::MiscObligation => {}
             ObligationCauseCode::SliceOrArrayElem => {
                 err.note("slice and array elements must have `Sized` type");
@@ -1908,7 +1911,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
 
                 if is_const_fn {
                     err.help(
-                        "consider creating a new `const` item and initializing with the result \
+                        "consider creating a new `const` item and initializing it with the result \
                         of the function call to be used in the repeat position, like \
                         `const VAL: Type = const_fn();` and `let x = [VAL; 42];`",
                     );
@@ -1916,7 +1919,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
 
                 if self.tcx.sess.is_nightly_build() && is_const_fn {
                     err.help(
-                        "create an inline `const` block, see PR \
+                        "create an inline `const` block, see RFC \
                         #2920 <https://github.com/rust-lang/rfcs/pull/2920> \
                         for more information",
                     );
@@ -2067,7 +2070,14 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
 
                 // Don't print the tuple of capture types
                 if !is_upvar_tys_infer_tuple {
-                    err.note(&format!("required because it appears within the type `{}`", ty));
+                    let msg = format!("required because it appears within the type `{}`", ty);
+                    match ty.kind() {
+                        ty::Adt(def, _) => match self.tcx.opt_item_name(def.did) {
+                            Some(ident) => err.span_note(ident.span, &msg),
+                            None => err.note(&msg),
+                        },
+                        _ => err.note(&msg),
+                    };
                 }
 
                 obligated_types.push(ty);
@@ -2089,11 +2099,36 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
             ObligationCauseCode::ImplDerivedObligation(ref data) => {
                 let mut parent_trait_ref = self.resolve_vars_if_possible(data.parent_trait_ref);
                 let parent_def_id = parent_trait_ref.def_id();
-                err.note(&format!(
+                let msg = format!(
                     "required because of the requirements on the impl of `{}` for `{}`",
                     parent_trait_ref.print_only_trait_path(),
                     parent_trait_ref.skip_binder().self_ty()
-                ));
+                );
+                let mut candidates = vec![];
+                self.tcx.for_each_relevant_impl(
+                    parent_def_id,
+                    parent_trait_ref.self_ty().skip_binder(),
+                    |impl_def_id| {
+                        candidates.push(impl_def_id);
+                    },
+                );
+                match &candidates[..] {
+                    [def_id] => match self.tcx.hir().get_if_local(*def_id) {
+                        Some(Node::Item(hir::Item {
+                            kind: hir::ItemKind::Impl(hir::Impl { of_trait, self_ty, .. }),
+                            ..
+                        })) => {
+                            let mut spans = Vec::with_capacity(2);
+                            if let Some(trait_ref) = of_trait {
+                                spans.push(trait_ref.path.span);
+                            }
+                            spans.push(self_ty.span);
+                            err.span_note(spans, &msg)
+                        }
+                        _ => err.note(&msg),
+                    },
+                    _ => err.note(&msg),
+                };
 
                 let mut parent_predicate = parent_trait_ref.without_const().to_predicate(tcx);
                 let mut data = data;
@@ -2144,19 +2179,60 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
                     )
                 });
             }
-            ObligationCauseCode::CompareImplMethodObligation { .. } => {
-                err.note(&format!(
-                    "the requirement `{}` appears on the impl method but not on the corresponding \
-                     trait method",
-                    predicate
-                ));
+            ObligationCauseCode::CompareImplMethodObligation {
+                item_name,
+                trait_item_def_id,
+                ..
+            } => {
+                let msg = format!(
+                    "the requirement `{}` appears on the impl method `{}` but not on the \
+                     corresponding trait method",
+                    predicate, item_name,
+                );
+                let sp = self
+                    .tcx
+                    .opt_item_name(trait_item_def_id)
+                    .map(|i| i.span)
+                    .unwrap_or_else(|| self.tcx.def_span(trait_item_def_id));
+                let mut assoc_span: MultiSpan = sp.into();
+                assoc_span.push_span_label(
+                    sp,
+                    format!("this trait method doesn't have the requirement `{}`", predicate),
+                );
+                if let Some(ident) = self
+                    .tcx
+                    .opt_associated_item(trait_item_def_id)
+                    .and_then(|i| self.tcx.opt_item_name(i.container.id()))
+                {
+                    assoc_span.push_span_label(ident.span, "in this trait".into());
+                }
+                err.span_note(assoc_span, &msg);
             }
-            ObligationCauseCode::CompareImplTypeObligation { .. } => {
-                err.note(&format!(
-                    "the requirement `{}` appears on the associated impl type but not on the \
+            ObligationCauseCode::CompareImplTypeObligation {
+                item_name, trait_item_def_id, ..
+            } => {
+                let msg = format!(
+                    "the requirement `{}` appears on the associated impl type `{}` but not on the \
                      corresponding associated trait type",
-                    predicate
-                ));
+                    predicate, item_name,
+                );
+                let sp = self.tcx.def_span(trait_item_def_id);
+                let mut assoc_span: MultiSpan = sp.into();
+                assoc_span.push_span_label(
+                    sp,
+                    format!(
+                        "this trait associated type doesn't have the requirement `{}`",
+                        predicate,
+                    ),
+                );
+                if let Some(ident) = self
+                    .tcx
+                    .opt_associated_item(trait_item_def_id)
+                    .and_then(|i| self.tcx.opt_item_name(i.container.id()))
+                {
+                    assoc_span.push_span_label(ident.span, "in this trait".into());
+                }
+                err.span_note(assoc_span, &msg);
             }
             ObligationCauseCode::CompareImplConstObligation => {
                 err.note(&format!(
@@ -2190,7 +2266,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> {
         &self,
         err: &mut DiagnosticBuilder<'_>,
         obligation: &PredicateObligation<'tcx>,
-        trait_ref: ty::Binder<ty::TraitRef<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
         span: Span,
     ) {
         debug!(
diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs
index 3c447a7d1f9..fc9739f70d4 100644
--- a/compiler/rustc_trait_selection/src/traits/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs
@@ -3,7 +3,8 @@ use rustc_data_structures::obligation_forest::ProcessResult;
 use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome};
 use rustc_data_structures::obligation_forest::{ObligationForest, ObligationProcessor};
 use rustc_errors::ErrorReported;
-use rustc_infer::traits::{TraitEngine, TraitEngineExt as _, TraitObligation};
+use rustc_infer::traits::{SelectionError, TraitEngine, TraitEngineExt as _, TraitObligation};
+use rustc_middle::mir::abstract_const::NotConstEvaluatable;
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::error::ExpectedFound;
 use rustc_middle::ty::subst::SubstsRef;
@@ -18,7 +19,7 @@ use super::wf;
 use super::CodeAmbiguity;
 use super::CodeProjectionError;
 use super::CodeSelectionError;
-use super::{ConstEvalFailure, Unimplemented};
+use super::Unimplemented;
 use super::{FulfillmentError, FulfillmentErrorCode};
 use super::{ObligationCause, PredicateObligation};
 
@@ -87,7 +88,7 @@ pub struct PendingPredicateObligation<'tcx> {
 }
 
 // `PendingPredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(target_arch = "x86_64")]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(PendingPredicateObligation<'_>, 56);
 
 impl<'a, 'tcx> FulfillmentContext<'tcx> {
@@ -498,14 +499,19 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> {
                         obligation.cause.span,
                     ) {
                         Ok(()) => ProcessResult::Changed(vec![]),
-                        Err(ErrorHandled::TooGeneric) => {
+                        Err(NotConstEvaluatable::MentionsInfer) => {
                             pending_obligation.stalled_on.clear();
                             pending_obligation.stalled_on.extend(
                                 substs.iter().filter_map(TyOrConstInferVar::maybe_from_generic_arg),
                             );
                             ProcessResult::Unchanged
                         }
-                        Err(e) => ProcessResult::Error(CodeSelectionError(ConstEvalFailure(e))),
+                        Err(
+                            e @ NotConstEvaluatable::MentionsParam
+                            | e @ NotConstEvaluatable::Error(_),
+                        ) => ProcessResult::Error(CodeSelectionError(
+                            SelectionError::NotConstEvaluatable(e),
+                        )),
                     }
                 }
 
@@ -516,15 +522,13 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> {
                         // if the constants depend on generic parameters.
                         //
                         // Let's just see where this breaks :shrug:
-                        if let (
-                            ty::ConstKind::Unevaluated(a_def, a_substs, None),
-                            ty::ConstKind::Unevaluated(b_def, b_substs, None),
-                        ) = (c1.val, c2.val)
+                        if let (ty::ConstKind::Unevaluated(a), ty::ConstKind::Unevaluated(b)) =
+                            (c1.val, c2.val)
                         {
                             if self
                                 .selcx
                                 .tcx()
-                                .try_unify_abstract_consts(((a_def, a_substs), (b_def, b_substs)))
+                                .try_unify_abstract_consts(((a.def, a.substs), (b.def, b.substs)))
                             {
                                 return ProcessResult::Changed(vec![]);
                             }
@@ -534,18 +538,17 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> {
                     let stalled_on = &mut pending_obligation.stalled_on;
 
                     let mut evaluate = |c: &'tcx Const<'tcx>| {
-                        if let ty::ConstKind::Unevaluated(def, substs, promoted) = c.val {
+                        if let ty::ConstKind::Unevaluated(unevaluated) = c.val {
                             match self.selcx.infcx().const_eval_resolve(
                                 obligation.param_env,
-                                def,
-                                substs,
-                                promoted,
+                                unevaluated,
                                 Some(obligation.cause.span),
                             ) {
                                 Ok(val) => Ok(Const::from_value(self.selcx.tcx(), val, c.ty)),
                                 Err(ErrorHandled::TooGeneric) => {
                                     stalled_on.extend(
-                                        substs
+                                        unevaluated
+                                            .substs
                                             .iter()
                                             .filter_map(TyOrConstInferVar::maybe_from_generic_arg),
                                     );
@@ -576,11 +579,11 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> {
                             }
                         }
                         (Err(ErrorHandled::Reported(ErrorReported)), _)
-                        | (_, Err(ErrorHandled::Reported(ErrorReported))) => {
-                            ProcessResult::Error(CodeSelectionError(ConstEvalFailure(
-                                ErrorHandled::Reported(ErrorReported),
-                            )))
-                        }
+                        | (_, Err(ErrorHandled::Reported(ErrorReported))) => ProcessResult::Error(
+                            CodeSelectionError(SelectionError::NotConstEvaluatable(
+                                NotConstEvaluatable::Error(ErrorReported),
+                            )),
+                        ),
                         (Err(ErrorHandled::Linted), _) | (_, Err(ErrorHandled::Linted)) => {
                             span_bug!(
                                 obligation.cause.span(self.selcx.tcx()),
@@ -681,7 +684,7 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> {
 /// Returns the set of inference variables contained in `substs`.
 fn substs_infer_vars<'a, 'tcx>(
     selcx: &mut SelectionContext<'a, 'tcx>,
-    substs: ty::Binder<SubstsRef<'tcx>>,
+    substs: ty::Binder<'tcx, SubstsRef<'tcx>>,
 ) -> impl Iterator<Item = TyOrConstInferVar<'tcx>> {
     selcx
         .infcx()
diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs
index 5a8c53a0c4b..f26eb159105 100644
--- a/compiler/rustc_trait_selection/src/traits/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/mod.rs
@@ -483,7 +483,7 @@ fn vtable_methods<'tcx>(
             let substs = trait_ref.map_bound(|trait_ref| {
                 InternalSubsts::for_item(tcx, def_id, |param, _| match param.kind {
                     GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
-                    GenericParamDefKind::Type { .. } | GenericParamDefKind::Const => {
+                    GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
                         trait_ref.substs[param.index as usize]
                     }
                 })
diff --git a/compiler/rustc_trait_selection/src/traits/object_safety.rs b/compiler/rustc_trait_selection/src/traits/object_safety.rs
index 7de20e477fe..ac5ec24eeee 100644
--- a/compiler/rustc_trait_selection/src/traits/object_safety.rs
+++ b/compiler/rustc_trait_selection/src/traits/object_safety.rs
@@ -439,8 +439,7 @@ fn virtual_call_violation_for_method<'tcx>(
         return Some(MethodViolationCode::WhereClauseReferencesSelf);
     }
 
-    let receiver_ty =
-        tcx.liberate_late_bound_regions(method.def_id, sig.map_bound(|sig| sig.inputs()[0]));
+    let receiver_ty = tcx.liberate_late_bound_regions(method.def_id, sig.input(0));
 
     // Until `unsized_locals` is fully implemented, `self: Self` can't be dispatched on.
     // However, this is already considered object-safe. We allow it as a special case here.
@@ -757,7 +756,7 @@ fn contains_illegal_self_type_reference<'tcx, T: TypeFoldable<'tcx>>(
     struct IllegalSelfTypeVisitor<'tcx> {
         tcx: TyCtxt<'tcx>,
         trait_def_id: DefId,
-        supertraits: Option<Vec<ty::PolyTraitRef<'tcx>>>,
+        supertraits: Option<Vec<DefId>>,
     }
 
     impl<'tcx> TypeVisitor<'tcx> for IllegalSelfTypeVisitor<'tcx> {
@@ -778,8 +777,10 @@ fn contains_illegal_self_type_reference<'tcx, T: TypeFoldable<'tcx>>(
                     // Compute supertraits of current trait lazily.
                     if self.supertraits.is_none() {
                         let trait_ref =
-                            ty::Binder::bind(ty::TraitRef::identity(self.tcx, self.trait_def_id));
-                        self.supertraits = Some(traits::supertraits(self.tcx, trait_ref).collect());
+                            ty::Binder::dummy(ty::TraitRef::identity(self.tcx, self.trait_def_id));
+                        self.supertraits = Some(
+                            traits::supertraits(self.tcx, trait_ref).map(|t| t.def_id()).collect(),
+                        );
                     }
 
                     // Determine whether the trait reference `Foo as
@@ -790,9 +791,11 @@ fn contains_illegal_self_type_reference<'tcx, T: TypeFoldable<'tcx>>(
                     // direct equality here because all of these types
                     // are part of the formal parameter listing, and
                     // hence there should be no inference variables.
-                    let projection_trait_ref = ty::Binder::bind(data.trait_ref(self.tcx));
-                    let is_supertrait_of_current_trait =
-                        self.supertraits.as_ref().unwrap().contains(&projection_trait_ref);
+                    let is_supertrait_of_current_trait = self
+                        .supertraits
+                        .as_ref()
+                        .unwrap()
+                        .contains(&data.trait_ref(self.tcx).def_id);
 
                     if is_supertrait_of_current_trait {
                         ControlFlow::CONTINUE // do not walk contained types, do not report error, do collect $200
diff --git a/compiler/rustc_trait_selection/src/traits/on_unimplemented.rs b/compiler/rustc_trait_selection/src/traits/on_unimplemented.rs
index 75822eadb2a..209fd83b3ab 100644
--- a/compiler/rustc_trait_selection/src/traits/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/traits/on_unimplemented.rs
@@ -337,7 +337,7 @@ impl<'tcx> OnUnimplementedFormatString {
             .iter()
             .filter_map(|param| {
                 let value = match param.kind {
-                    GenericParamDefKind::Type { .. } | GenericParamDefKind::Const => {
+                    GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
                         trait_ref.substs[param.index as usize].to_string()
                     }
                     GenericParamDefKind::Lifetime => return None,
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index 0af6d645915..b3e5df4da0a 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -1275,6 +1275,9 @@ fn confirm_discriminant_kind_candidate<'cx, 'tcx>(
     let tcx = selcx.tcx();
 
     let self_ty = selcx.infcx().shallow_resolve(obligation.predicate.self_ty());
+    // We get here from `poly_project_and_unify_type` which replaces bound vars
+    // with placeholders
+    debug_assert!(!self_ty.has_escaping_bound_vars());
     let substs = tcx.mk_substs([self_ty.into()].iter());
 
     let discriminant_def_id = tcx.require_lang_item(LangItem::Discriminant, None);
@@ -1306,7 +1309,7 @@ fn confirm_pointee_candidate<'cx, 'tcx>(
         ty: self_ty.ptr_metadata_ty(tcx),
     };
 
-    confirm_param_env_candidate(selcx, obligation, ty::Binder::bind(predicate), false)
+    confirm_param_env_candidate(selcx, obligation, ty::Binder::bind(predicate, tcx), false)
 }
 
 fn confirm_fn_pointer_candidate<'cx, 'tcx>(
diff --git a/compiler/rustc_trait_selection/src/traits/query/mod.rs b/compiler/rustc_trait_selection/src/traits/query/mod.rs
index 01f4f09e238..f6f42814d3f 100644
--- a/compiler/rustc_trait_selection/src/traits/query/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/mod.rs
@@ -3,7 +3,7 @@
 //! which makes a canonical query by replacing unbound inference
 //! variables and regions, so that results can be reused more broadly.
 //! The providers for the queries defined here can be found in
-//! `librustc_traits`.
+//! `rustc_traits`.
 
 pub mod dropck_outlives;
 pub mod evaluate_obligation;
diff --git a/compiler/rustc_trait_selection/src/traits/query/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
index c908e1418c1..eb7ea8715c2 100644
--- a/compiler/rustc_trait_selection/src/traits/query/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
@@ -10,6 +10,7 @@ use crate::traits::{Obligation, ObligationCause, PredicateObligation, Reveal};
 use rustc_data_structures::sso::SsoHashMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_infer::traits::Normalized;
+use rustc_middle::mir;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder};
 use rustc_middle::ty::subst::Subst;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -214,4 +215,8 @@ impl<'cx, 'tcx> TypeFolder<'tcx> for QueryNormalizer<'cx, 'tcx> {
         let constant = constant.super_fold_with(self);
         constant.eval(self.infcx.tcx, self.param_env)
     }
+
+    fn fold_mir_const(&mut self, constant: mir::ConstantKind<'tcx>) -> mir::ConstantKind<'tcx> {
+        constant.super_fold_with(self)
+    }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
index 68356ce73aa..4f5476ca5d0 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
@@ -62,7 +62,7 @@ fn scrape_region_constraints<'tcx, R>(
     infcx: &InferCtxt<'_, 'tcx>,
     op: impl FnOnce() -> Fallible<InferOk<'tcx, R>>,
 ) -> Fallible<(R, Option<Rc<QueryRegionConstraints<'tcx>>>)> {
-    let mut fulfill_cx = TraitEngine::new(infcx.tcx);
+    let mut fulfill_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
     let dummy_body_id = ObligationCause::dummy().body_id;
 
     // During NLL, we expect that nobody will register region
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
index cf7f0a553c7..b351af44e94 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
@@ -8,12 +8,6 @@ pub struct ImpliedOutlivesBounds<'tcx> {
     pub ty: Ty<'tcx>,
 }
 
-impl<'tcx> ImpliedOutlivesBounds<'tcx> {
-    pub fn new(ty: Ty<'tcx>) -> Self {
-        ImpliedOutlivesBounds { ty }
-    }
-}
-
 impl<'tcx> super::QueryTypeOp<'tcx> for ImpliedOutlivesBounds<'tcx> {
     type QueryResponse = Vec<OutlivesBound<'tcx>>;
 
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 272930f6bb9..31685a012ca 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -272,7 +272,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         &mut self,
         obligation: &TraitObligation<'tcx>,
         trait_def_id: DefId,
-        nested: ty::Binder<Vec<Ty<'tcx>>>,
+        nested: ty::Binder<'tcx, Vec<Ty<'tcx>>>,
     ) -> ImplSourceAutoImplData<PredicateObligation<'tcx>> {
         debug!(?nested, "vtable_auto_impl");
         ensure_sufficient_stack(|| {
@@ -748,7 +748,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     cause,
                     obligation.recursion_depth + 1,
                     obligation.param_env,
-                    ty::Binder::bind(outlives).to_predicate(tcx),
+                    obligation.predicate.rebind(outlives).to_predicate(tcx),
                 ));
             }
 
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 45b5aff40a6..e4aabbdb7ed 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -34,6 +34,7 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::Constness;
 use rustc_infer::infer::LateBoundRegionConversionTime;
 use rustc_middle::dep_graph::{DepKind, DepNodeIndex};
+use rustc_middle::mir::abstract_const::NotConstEvaluatable;
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::fast_reject;
 use rustc_middle::ty::print::with_no_trimmed_paths;
@@ -203,7 +204,7 @@ struct EvaluatedCandidate<'tcx> {
 /// When does the builtin impl for `T: Trait` apply?
 enum BuiltinImplConditions<'tcx> {
     /// The impl is conditional on `T1, T2, ...: Trait`.
-    Where(ty::Binder<Vec<Ty<'tcx>>>),
+    Where(ty::Binder<'tcx, Vec<Ty<'tcx>>>),
     /// There is no built-in impl. There may be some other
     /// candidate (a where-clause or user-defined impl).
     None,
@@ -547,7 +548,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                         obligation.cause.span,
                     ) {
                         Ok(()) => Ok(EvaluatedToOk),
-                        Err(ErrorHandled::TooGeneric) => Ok(EvaluatedToAmbig),
+                        Err(NotConstEvaluatable::MentionsInfer) => Ok(EvaluatedToAmbig),
+                        Err(NotConstEvaluatable::MentionsParam) => Ok(EvaluatedToErr),
                         Err(_) => Ok(EvaluatedToErr),
                     }
                 }
@@ -556,13 +558,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     debug!(?c1, ?c2, "evaluate_predicate_recursively: equating consts");
 
                     let evaluate = |c: &'tcx ty::Const<'tcx>| {
-                        if let ty::ConstKind::Unevaluated(def, substs, promoted) = c.val {
+                        if let ty::ConstKind::Unevaluated(unevaluated) = c.val {
                             self.infcx
                                 .const_eval_resolve(
                                     obligation.param_env,
-                                    def,
-                                    substs,
-                                    promoted,
+                                    unevaluated,
                                     Some(obligation.cause.span),
                                 )
                                 .map(|val| ty::Const::from_value(self.tcx(), val, c.ty))
@@ -863,7 +863,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         stack: &TraitObligationStack<'o, 'tcx>,
         candidate: &SelectionCandidate<'tcx>,
     ) -> Result<EvaluationResult, OverflowError> {
-        let result = self.evaluation_probe(|this| {
+        let mut result = self.evaluation_probe(|this| {
             let candidate = (*candidate).clone();
             match this.confirm_candidate(stack.obligation, candidate) {
                 Ok(selection) => {
@@ -876,6 +876,19 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 Err(..) => Ok(EvaluatedToErr),
             }
         })?;
+
+        // If we erased any lifetimes, then we want to use
+        // `EvaluatedToOkModuloRegions` instead of `EvaluatedToOk`
+        // as your final result. The result will be cached using
+        // the freshened trait predicate as a key, so we need
+        // our result to be correct by *any* choice of original lifetimes,
+        // not just the lifetime choice for this particular (non-erased)
+        // predicate.
+        // See issue #80691
+        if stack.fresh_trait_ref.has_erased_regions() {
+            result = result.max(EvaluatedToOkModuloRegions);
+        }
+
         debug!(?result);
         Ok(result)
     }
@@ -931,7 +944,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     /// subobligations without taking in a 'parent' depth, causing the
     /// generated subobligations to have a `recursion_depth` of `0`.
     ///
-    /// To ensure that obligation_depth never decreasees, we force all subobligations
+    /// To ensure that obligation_depth never decreases, we force all subobligations
     /// to have at least the depth of the original obligation.
     fn add_depth<T: 'cx, I: Iterator<Item = &'cx mut Obligation<'tcx, T>>>(
         &self,
@@ -968,7 +981,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         OP: FnOnce(&mut Self) -> R,
     {
         let (result, dep_node) =
-            self.tcx().dep_graph.with_anon_task(DepKind::TraitSelect, || op(self));
+            self.tcx().dep_graph.with_anon_task(self.tcx(), DepKind::TraitSelect, || op(self));
         self.tcx().dep_graph.read_index(dep_node);
         (result, dep_node)
     }
@@ -1660,7 +1673,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     /// Bar<i32> where struct Bar<T> { x: T, y: u32 } -> [i32, u32]
     /// Zed<i32> where enum Zed { A(T), B(u32) } -> [i32, u32]
     /// ```
-    fn constituent_types_for_ty(&self, t: ty::Binder<Ty<'tcx>>) -> ty::Binder<Vec<Ty<'tcx>>> {
+    fn constituent_types_for_ty(
+        &self,
+        t: ty::Binder<'tcx, Ty<'tcx>>,
+    ) -> ty::Binder<'tcx, Vec<Ty<'tcx>>> {
         match *t.skip_binder().kind() {
             ty::Uint(_)
             | ty::Int(_)
@@ -1733,7 +1749,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         cause: ObligationCause<'tcx>,
         recursion_depth: usize,
         trait_def_id: DefId,
-        types: ty::Binder<Vec<Ty<'tcx>>>,
+        types: ty::Binder<'tcx, Vec<Ty<'tcx>>>,
     ) -> Vec<PredicateObligation<'tcx>> {
         // Because the types were potentially derived from
         // higher-ranked obligations they may reference late-bound
@@ -1754,7 +1770,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             .skip_binder() // binder moved -\
             .iter()
             .flat_map(|ty| {
-                let ty: ty::Binder<Ty<'tcx>> = types.rebind(ty); // <----/
+                let ty: ty::Binder<'tcx, Ty<'tcx>> = types.rebind(ty); // <----/
 
                 self.infcx.commit_unconditionally(|_| {
                     let placeholder_ty = self.infcx.replace_bound_vars_with_placeholders(ty);
@@ -1874,7 +1890,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         // substitution if we find that any of the input types, when
         // simplified, do not match.
 
-        obligation.predicate.skip_binder().trait_ref.substs.iter().zip(impl_trait_ref.substs).any(
+        iter::zip(obligation.predicate.skip_binder().trait_ref.substs, impl_trait_ref.substs).any(
             |(obligation_arg, impl_arg)| {
                 match (obligation_arg.unpack(), impl_arg.unpack()) {
                     (GenericArgKind::Type(obligation_ty), GenericArgKind::Type(impl_ty)) => {
diff --git a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
index 264cc4f248c..4b563a87a15 100644
--- a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
@@ -395,14 +395,14 @@ fn report_conflicting_impls(
     // that's passed in.
     let decorate = |err: LintDiagnosticBuilder<'_>| {
         let msg = format!(
-            "conflicting implementations of trait `{}`{}:{}",
+            "conflicting implementations of trait `{}`{}{}",
             overlap.trait_desc,
             overlap
                 .self_desc
                 .clone()
                 .map_or_else(String::new, |ty| { format!(" for type `{}`", ty) }),
             match used_to_be_allowed {
-                Some(FutureCompatOverlapErrorKind::Issue33140) => " (E0119)",
+                Some(FutureCompatOverlapErrorKind::Issue33140) => ": (E0119)",
                 _ => "",
             }
         );
diff --git a/compiler/rustc_trait_selection/src/traits/util.rs b/compiler/rustc_trait_selection/src/traits/util.rs
index 8888ea2c849..fd94f9f7998 100644
--- a/compiler/rustc_trait_selection/src/traits/util.rs
+++ b/compiler/rustc_trait_selection/src/traits/util.rs
@@ -328,7 +328,7 @@ pub fn closure_trait_ref_and_return_type(
     self_ty: Ty<'tcx>,
     sig: ty::PolyFnSig<'tcx>,
     tuple_arguments: TupleArgumentsFlag,
-) -> ty::Binder<(ty::TraitRef<'tcx>, Ty<'tcx>)> {
+) -> ty::Binder<'tcx, (ty::TraitRef<'tcx>, Ty<'tcx>)> {
     let arguments_tuple = match tuple_arguments {
         TupleArgumentsFlag::No => sig.skip_binder().inputs()[0],
         TupleArgumentsFlag::Yes => tcx.intern_tup(sig.skip_binder().inputs()),
@@ -346,7 +346,7 @@ pub fn generator_trait_ref_and_outputs(
     fn_trait_def_id: DefId,
     self_ty: Ty<'tcx>,
     sig: ty::PolyGenSig<'tcx>,
-) -> ty::Binder<(ty::TraitRef<'tcx>, Ty<'tcx>, Ty<'tcx>)> {
+) -> ty::Binder<'tcx, (ty::TraitRef<'tcx>, Ty<'tcx>, Ty<'tcx>)> {
     debug_assert!(!self_ty.has_escaping_bound_vars());
     let trait_ref = ty::TraitRef {
         def_id: fn_trait_def_id,
diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs
index e6ef9b137d8..f592cf1cd24 100644
--- a/compiler/rustc_trait_selection/src/traits/wf.rs
+++ b/compiler/rustc_trait_selection/src/traits/wf.rs
@@ -430,7 +430,7 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> {
 
                 GenericArgKind::Const(constant) => {
                     match constant.val {
-                        ty::ConstKind::Unevaluated(def, substs, promoted) => {
+                        ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) => {
                             assert!(promoted.is_none());
 
                             let obligations = self.nominal_obligations(def.did, substs);
@@ -692,11 +692,7 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> {
         let predicates = predicates.instantiate(self.infcx.tcx, substs);
         debug_assert_eq!(predicates.predicates.len(), origins.len());
 
-        predicates
-            .predicates
-            .into_iter()
-            .zip(predicates.spans.into_iter())
-            .zip(origins.into_iter().rev())
+        iter::zip(iter::zip(predicates.predicates, predicates.spans), origins.into_iter().rev())
             .map(|((pred, span), origin_def_id)| {
                 let cause = self.cause(traits::BindingObligation(origin_def_id, span));
                 traits::Obligation::with_depth(cause, self.recursion_depth, self.param_env, pred)
@@ -708,7 +704,7 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> {
     fn from_object_ty(
         &mut self,
         ty: Ty<'tcx>,
-        data: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        data: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
         region: ty::Region<'tcx>,
     ) {
         // Imagine a type like this:
@@ -771,7 +767,7 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> {
 /// `infer::required_region_bounds`, see that for more information.
 pub fn object_region_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
-    existential_predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+    existential_predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
 ) -> Vec<ty::Region<'tcx>> {
     // Since we don't actually *know* the self type for an object,
     // this "open(err)" serves as a kind of dummy standin -- basically
diff --git a/compiler/rustc_traits/src/chalk/db.rs b/compiler/rustc_traits/src/chalk/db.rs
index 916186f4204..8c97e606c56 100644
--- a/compiler/rustc_traits/src/chalk/db.rs
+++ b/compiler/rustc_traits/src/chalk/db.rs
@@ -735,11 +735,14 @@ fn bound_vars_for_item(tcx: TyCtxt<'tcx>, def_id: DefId) -> SubstsRef<'tcx> {
             .into(),
 
         ty::GenericParamDefKind::Lifetime => {
-            let br = ty::BoundRegion { kind: ty::BrAnon(substs.len() as u32) };
+            let br = ty::BoundRegion {
+                var: ty::BoundVar::from_usize(substs.len()),
+                kind: ty::BrAnon(substs.len() as u32),
+            };
             tcx.mk_region(ty::RegionKind::ReLateBound(ty::INNERMOST, br)).into()
         }
 
-        ty::GenericParamDefKind::Const => tcx
+        ty::GenericParamDefKind::Const { .. } => tcx
             .mk_const(ty::Const {
                 val: ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from(param.index)),
                 ty: tcx.type_of(param.def_id),
diff --git a/compiler/rustc_traits/src/chalk/lowering.rs b/compiler/rustc_traits/src/chalk/lowering.rs
index fdf5f697e61..39890fd5b05 100644
--- a/compiler/rustc_traits/src/chalk/lowering.rs
+++ b/compiler/rustc_traits/src/chalk/lowering.rs
@@ -434,17 +434,11 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Lifetime<RustInterner<'tcx>>> for Region<'t
             ReEarlyBound(_) => {
                 panic!("Should have already been substituted.");
             }
-            ReLateBound(db, br) => match br.kind {
-                ty::BoundRegionKind::BrAnon(var) => {
-                    chalk_ir::LifetimeData::BoundVar(chalk_ir::BoundVar::new(
-                        chalk_ir::DebruijnIndex::new(db.as_u32()),
-                        var as usize,
-                    ))
-                    .intern(interner)
-                }
-                ty::BoundRegionKind::BrNamed(_def_id, _name) => unimplemented!(),
-                ty::BrEnv => unimplemented!(),
-            },
+            ReLateBound(db, br) => chalk_ir::LifetimeData::BoundVar(chalk_ir::BoundVar::new(
+                chalk_ir::DebruijnIndex::new(db.as_u32()),
+                br.var.as_usize(),
+            ))
+            .intern(interner),
             ReFree(_) => unimplemented!(),
             ReStatic => chalk_ir::LifetimeData::Static.intern(interner),
             ReVar(_) => unimplemented!(),
@@ -467,7 +461,10 @@ impl<'tcx> LowerInto<'tcx, Region<'tcx>> for &chalk_ir::Lifetime<RustInterner<'t
         let kind = match self.data(interner) {
             chalk_ir::LifetimeData::BoundVar(var) => ty::RegionKind::ReLateBound(
                 ty::DebruijnIndex::from_u32(var.debruijn.depth()),
-                ty::BoundRegion { kind: ty::BrAnon(var.index as u32) },
+                ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(var.index),
+                    kind: ty::BrAnon(var.index as u32),
+                },
             ),
             chalk_ir::LifetimeData::InferenceVar(_var) => unimplemented!(),
             chalk_ir::LifetimeData::Placeholder(p) => {
@@ -606,7 +603,7 @@ impl<'tcx> LowerInto<'tcx, Option<chalk_ir::QuantifiedWhereClause<RustInterner<'
 }
 
 impl<'tcx> LowerInto<'tcx, chalk_ir::Binders<chalk_ir::QuantifiedWhereClauses<RustInterner<'tcx>>>>
-    for &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>
+    for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>
 {
     fn lower_into(
         self,
@@ -677,7 +674,9 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Binders<chalk_ir::QuantifiedWhereClauses<Ru
     }
 }
 
-impl<'tcx> LowerInto<'tcx, chalk_ir::FnSig<RustInterner<'tcx>>> for ty::Binder<ty::FnSig<'tcx>> {
+impl<'tcx> LowerInto<'tcx, chalk_ir::FnSig<RustInterner<'tcx>>>
+    for ty::Binder<'tcx, ty::FnSig<'tcx>>
+{
     fn lower_into(self, _interner: &RustInterner<'_>) -> FnSig<RustInterner<'tcx>> {
         chalk_ir::FnSig {
             abi: self.abi(),
@@ -801,7 +800,7 @@ impl<'tcx> LowerInto<'tcx, chalk_solve::rust_ir::AliasEqBound<RustInterner<'tcx>
 crate fn collect_bound_vars<'tcx, T: TypeFoldable<'tcx>>(
     interner: &RustInterner<'tcx>,
     tcx: TyCtxt<'tcx>,
-    ty: Binder<T>,
+    ty: Binder<'tcx, T>,
 ) -> (T, chalk_ir::VariableKinds<RustInterner<'tcx>>, BTreeMap<DefId, u32>) {
     let mut bound_vars_collector = BoundVarsCollector::new();
     ty.as_ref().skip_binder().visit_with(&mut bound_vars_collector);
@@ -849,7 +848,10 @@ impl<'tcx> BoundVarsCollector<'tcx> {
 }
 
 impl<'tcx> TypeVisitor<'tcx> for BoundVarsCollector<'tcx> {
-    fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<Self::BreakTy> {
+    fn visit_binder<T: TypeFoldable<'tcx>>(
+        &mut self,
+        t: &Binder<'tcx, T>,
+    ) -> ControlFlow<Self::BreakTy> {
         self.binder_index.shift_in(1);
         let result = t.super_visit_with(self);
         self.binder_index.shift_out(1);
@@ -895,7 +897,7 @@ impl<'tcx> TypeVisitor<'tcx> for BoundVarsCollector<'tcx> {
                     },
                 },
 
-                ty::BrEnv => unimplemented!(),
+                ty::BoundRegionKind::BrEnv => unimplemented!(),
             },
 
             ty::ReEarlyBound(_re) => {
@@ -931,7 +933,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for NamedBoundVarSubstitutor<'a, 'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: Binder<T>) -> Binder<T> {
+    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: Binder<'tcx, T>) -> Binder<'tcx, T> {
         self.binder_index.shift_in(1);
         let result = t.super_fold_with(self);
         self.binder_index.shift_out(1);
@@ -943,7 +945,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for NamedBoundVarSubstitutor<'a, 'tcx> {
             ty::ReLateBound(index, br) if *index == self.binder_index => match br.kind {
                 ty::BrNamed(def_id, _name) => match self.named_parameters.get(&def_id) {
                     Some(idx) => {
-                        let new_br = ty::BoundRegion { kind: ty::BrAnon(*idx) };
+                        let new_br = ty::BoundRegion { var: br.var, kind: ty::BrAnon(*idx) };
                         return self.tcx.mk_region(RegionKind::ReLateBound(*index, new_br));
                     }
                     None => panic!("Missing `BrNamed`."),
@@ -987,7 +989,7 @@ impl<'tcx> TypeFolder<'tcx> for ParamsSubstitutor<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: Binder<T>) -> Binder<T> {
+    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: Binder<'tcx, T>) -> Binder<'tcx, T> {
         self.binder_index.shift_in(1);
         let result = t.super_fold_with(self);
         self.binder_index.shift_out(1);
@@ -1026,12 +1028,16 @@ impl<'tcx> TypeFolder<'tcx> for ParamsSubstitutor<'tcx> {
             // This covers any region variables in a goal, right?
             ty::ReEarlyBound(_re) => match self.named_regions.get(&_re.def_id) {
                 Some(idx) => {
-                    let br = ty::BoundRegion { kind: ty::BrAnon(*idx) };
+                    let br = ty::BoundRegion {
+                        var: ty::BoundVar::from_u32(*idx),
+                        kind: ty::BrAnon(*idx),
+                    };
                     self.tcx.mk_region(RegionKind::ReLateBound(self.binder_index, br))
                 }
                 None => {
                     let idx = self.named_regions.len() as u32;
-                    let br = ty::BoundRegion { kind: ty::BrAnon(idx) };
+                    let br =
+                        ty::BoundRegion { var: ty::BoundVar::from_u32(idx), kind: ty::BrAnon(idx) };
                     self.named_regions.insert(_re.def_id, idx);
                     self.tcx.mk_region(RegionKind::ReLateBound(self.binder_index, br))
                 }
diff --git a/compiler/rustc_traits/src/chalk/mod.rs b/compiler/rustc_traits/src/chalk/mod.rs
index d98f18182c8..b7275bac190 100644
--- a/compiler/rustc_traits/src/chalk/mod.rs
+++ b/compiler/rustc_traits/src/chalk/mod.rs
@@ -165,7 +165,7 @@ crate fn evaluate_goal<'tcx>(
                         // let's just ignore that
                         let sol = Canonical {
                             max_universe: ty::UniverseIndex::from_usize(0),
-                            variables: obligation.variables.clone(),
+                            variables: obligation.variables,
                             value: QueryResponse {
                                 var_values: CanonicalVarValues { var_values: IndexVec::new() }
                                     .make_identity(tcx),
diff --git a/compiler/rustc_traits/src/dropck_outlives.rs b/compiler/rustc_traits/src/dropck_outlives.rs
index cfcbc77c172..9d5b9d7357f 100644
--- a/compiler/rustc_traits/src/dropck_outlives.rs
+++ b/compiler/rustc_traits/src/dropck_outlives.rs
@@ -75,7 +75,7 @@ fn dropck_outlives<'tcx>(
             // Set used to detect infinite recursion.
             let mut ty_set = FxHashSet::default();
 
-            let mut fulfill_cx = TraitEngine::new(infcx.tcx);
+            let mut fulfill_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
 
             let cause = ObligationCause::dummy();
             let mut constraints = DtorckConstraint::empty();
diff --git a/compiler/rustc_traits/src/normalize_erasing_regions.rs b/compiler/rustc_traits/src/normalize_erasing_regions.rs
index 1213e553908..5ad0684fe6e 100644
--- a/compiler/rustc_traits/src/normalize_erasing_regions.rs
+++ b/compiler/rustc_traits/src/normalize_erasing_regions.rs
@@ -1,24 +1,35 @@
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::traits::query::NoSolution;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::subst::GenericArg;
 use rustc_middle::ty::{self, ParamEnvAnd, TyCtxt, TypeFoldable};
 use rustc_trait_selection::traits::query::normalize::AtExt;
 use rustc_trait_selection::traits::{Normalized, ObligationCause};
 use std::sync::atomic::Ordering;
 
 crate fn provide(p: &mut Providers) {
-    *p = Providers { normalize_generic_arg_after_erasing_regions, ..*p };
+    *p = Providers {
+        normalize_generic_arg_after_erasing_regions: |tcx, goal| {
+            debug!("normalize_generic_arg_after_erasing_regions(goal={:#?})", goal);
+
+            tcx.sess
+                .perf_stats
+                .normalize_generic_arg_after_erasing_regions
+                .fetch_add(1, Ordering::Relaxed);
+            normalize_after_erasing_regions(tcx, goal)
+        },
+        normalize_mir_const_after_erasing_regions: |tcx, goal| {
+            normalize_after_erasing_regions(tcx, goal)
+        },
+        ..*p
+    };
 }
 
-fn normalize_generic_arg_after_erasing_regions<'tcx>(
+#[instrument(level = "debug", skip(tcx))]
+fn normalize_after_erasing_regions<'tcx, T: TypeFoldable<'tcx> + PartialEq + Copy>(
     tcx: TyCtxt<'tcx>,
-    goal: ParamEnvAnd<'tcx, GenericArg<'tcx>>,
-) -> GenericArg<'tcx> {
-    debug!("normalize_generic_arg_after_erasing_regions(goal={:#?})", goal);
-
+    goal: ParamEnvAnd<'tcx, T>,
+) -> T {
     let ParamEnvAnd { param_env, value } = goal;
-    tcx.sess.perf_stats.normalize_generic_arg_after_erasing_regions.fetch_add(1, Ordering::Relaxed);
     tcx.infer_ctxt().enter(|infcx| {
         let cause = ObligationCause::dummy();
         match infcx.at(&cause, param_env).normalize(value) {
diff --git a/compiler/rustc_ty_utils/src/common_traits.rs b/compiler/rustc_ty_utils/src/common_traits.rs
index 24ba0717866..cedc84d97c2 100644
--- a/compiler/rustc_ty_utils/src/common_traits.rs
+++ b/compiler/rustc_ty_utils/src/common_traits.rs
@@ -18,6 +18,10 @@ fn is_freeze_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>
     is_item_raw(tcx, query, LangItem::Freeze)
 }
 
+fn is_unpin_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
+    is_item_raw(tcx, query, LangItem::Unpin)
+}
+
 fn is_item_raw<'tcx>(
     tcx: TyCtxt<'tcx>,
     query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
@@ -37,5 +41,11 @@ fn is_item_raw<'tcx>(
 }
 
 pub(crate) fn provide(providers: &mut ty::query::Providers) {
-    *providers = ty::query::Providers { is_copy_raw, is_sized_raw, is_freeze_raw, ..*providers };
+    *providers = ty::query::Providers {
+        is_copy_raw,
+        is_sized_raw,
+        is_freeze_raw,
+        is_unpin_raw,
+        ..*providers
+    };
 }
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 6b9d46ee0af..874289d0293 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -115,7 +115,7 @@ fn resolve_associated_item<'tcx>(
     );
 
     let trait_ref = ty::TraitRef::from_method(tcx, trait_id, rcvr_substs);
-    let vtbl = tcx.codegen_fulfill_obligation((param_env, ty::Binder::bind(trait_ref)))?;
+    let vtbl = tcx.codegen_fulfill_obligation((param_env, ty::Binder::bind(trait_ref, tcx)))?;
 
     // Now that we know which impl is being used, we can dispatch to
     // the actual function:
diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs
index 29f1761b84d..38e5ce6fd83 100644
--- a/compiler/rustc_ty_utils/src/ty.rs
+++ b/compiler/rustc_ty_utils/src/ty.rs
@@ -210,9 +210,9 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] {
     }
 }
 
-fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssociatedItems<'_> {
+fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssocItems<'_> {
     let items = tcx.associated_item_def_ids(def_id).iter().map(|did| tcx.associated_item(*did));
-    ty::AssociatedItems::new(items)
+    ty::AssocItems::new(items)
 }
 
 fn def_ident_span(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Span> {
diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs
index fccd8b795ef..8fcdf813b41 100644
--- a/compiler/rustc_type_ir/src/lib.rs
+++ b/compiler/rustc_type_ir/src/lib.rs
@@ -1,7 +1,3 @@
-#![feature(never_type)]
-#![feature(const_panic)]
-#![feature(control_flow_enum)]
-
 #[macro_use]
 extern crate bitflags;
 #[macro_use]
diff --git a/compiler/rustc_typeck/src/astconv/errors.rs b/compiler/rustc_typeck/src/astconv/errors.rs
index b5404c3a15c..695132281c6 100644
--- a/compiler/rustc_typeck/src/astconv/errors.rs
+++ b/compiler/rustc_typeck/src/astconv/errors.rs
@@ -10,6 +10,7 @@ use rustc_span::symbol::{sym, Ident};
 use rustc_span::{Span, DUMMY_SP};
 
 use std::collections::BTreeSet;
+use std::iter;
 
 impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     /// On missing type parameters, emit an E0393 error and provide a structured suggestion using
@@ -309,7 +310,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 // that the user forgot to give the associtated type's name. The canonical
                 // example would be trying to use `Iterator<isize>` instead of
                 // `Iterator<Item = isize>`.
-                for (potential, item) in potential_assoc_types.iter().zip(assoc_items.iter()) {
+                for (potential, item) in iter::zip(&potential_assoc_types, assoc_items) {
                     if let Ok(snippet) = tcx.sess.source_map().span_to_snippet(*potential) {
                         suggestions.push((*potential, format!("{} = {}", item.ident, snippet)));
                     }
diff --git a/compiler/rustc_typeck/src/astconv/generics.rs b/compiler/rustc_typeck/src/astconv/generics.rs
index 0ea0ccaceab..7a297f2c65f 100644
--- a/compiler/rustc_typeck/src/astconv/generics.rs
+++ b/compiler/rustc_typeck/src/astconv/generics.rs
@@ -64,7 +64,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     kind: hir::TyKind::Path(rustc_hir::QPath::Resolved(_, path)),
                     ..
                 }),
-                GenericParamDefKind::Const,
+                GenericParamDefKind::Const { .. },
             ) => match path.res {
                 Res::Err => {
                     add_braces_suggestion(arg, &mut err);
@@ -82,7 +82,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         if param_type.is_suggestable() {
                             err.span_suggestion(
                                 tcx.def_span(src_def_id),
-                                "consider changing this type paramater to a `const`-generic",
+                                "consider changing this type parameter to be a `const` generic",
                                 format!("const {}: {}", param_name, param_type),
                                 Applicability::MaybeIncorrect,
                             );
@@ -93,7 +93,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             },
             (
                 GenericArg::Type(hir::Ty { kind: hir::TyKind::Path(_), .. }),
-                GenericParamDefKind::Const,
+                GenericParamDefKind::Const { .. },
             ) => add_braces_suggestion(arg, &mut err),
             (
                 GenericArg::Type(hir::Ty { kind: hir::TyKind::Array(_, len), .. }),
@@ -166,7 +166,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         parent_substs: &[subst::GenericArg<'tcx>],
         has_self: bool,
         self_ty: Option<Ty<'tcx>>,
-        arg_count: GenericArgCountResult,
+        arg_count: &GenericArgCountResult,
         ctx: &mut impl CreateSubstsForGenericArgsCtxt<'a, 'tcx>,
     ) -> SubstsRef<'tcx> {
         // Collect the segments of the path; we need to substitute arguments
@@ -236,7 +236,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         match (arg, &param.kind, arg_count.explicit_late_bound) {
                             (GenericArg::Lifetime(_), GenericParamDefKind::Lifetime, _)
                             | (GenericArg::Type(_), GenericParamDefKind::Type { .. }, _)
-                            | (GenericArg::Const(_), GenericParamDefKind::Const, _) => {
+                            | (GenericArg::Const(_), GenericParamDefKind::Const { .. }, _) => {
                                 substs.push(ctx.provided_kind(param, arg));
                                 args.next();
                                 params.next();
@@ -282,7 +282,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                                                     GenericParamDefKind::Type { .. } => {
                                                         ParamKindOrd::Type
                                                     }
-                                                    GenericParamDefKind::Const => {
+                                                    GenericParamDefKind::Const { .. } => {
                                                         ParamKindOrd::Const {
                                                             unordered: tcx
                                                                 .features()
@@ -499,7 +499,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             let expected_min = if infer_args {
                 0
             } else {
-                param_counts.consts + named_type_param_count - default_counts.types
+                param_counts.consts + named_type_param_count
+                    - default_counts.types
+                    - default_counts.consts
             };
 
             check_generics(
diff --git a/compiler/rustc_typeck/src/astconv/mod.rs b/compiler/rustc_typeck/src/astconv/mod.rs
index 947363fc3ed..b6de491911a 100644
--- a/compiler/rustc_typeck/src/astconv/mod.rs
+++ b/compiler/rustc_typeck/src/astconv/mod.rs
@@ -198,6 +198,7 @@ pub trait CreateSubstsForGenericArgsCtxt<'a, 'tcx> {
 }
 
 impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
+    #[tracing::instrument(level = "debug", skip(self))]
     pub fn ast_region_to_region(
         &self,
         lifetime: &hir::Lifetime,
@@ -209,14 +210,20 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         let r = match tcx.named_region(lifetime.hir_id) {
             Some(rl::Region::Static) => tcx.lifetimes.re_static,
 
-            Some(rl::Region::LateBound(debruijn, id, _)) => {
-                let name = lifetime_name(id.expect_local());
-                let br = ty::BoundRegion { kind: ty::BrNamed(id, name) };
+            Some(rl::Region::LateBound(debruijn, index, def_id, _)) => {
+                let name = lifetime_name(def_id.expect_local());
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_u32(index),
+                    kind: ty::BrNamed(def_id, name),
+                };
                 tcx.mk_region(ty::ReLateBound(debruijn, br))
             }
 
-            Some(rl::Region::LateBoundAnon(debruijn, index)) => {
-                let br = ty::BoundRegion { kind: ty::BrAnon(index) };
+            Some(rl::Region::LateBoundAnon(debruijn, index, anon_index)) => {
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_u32(index),
+                    kind: ty::BrAnon(anon_index),
+                };
                 tcx.mk_region(ty::ReLateBound(debruijn, br))
             }
 
@@ -237,6 +244,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
             None => {
                 self.re_infer(def, lifetime.span).unwrap_or_else(|| {
+                    debug!(?lifetime, "unelided lifetime in signature");
+
                     // This indicates an illegal lifetime
                     // elision. `resolve_lifetime` should have
                     // reported an error in this case -- but if
@@ -263,7 +272,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         def_id: DefId,
         item_segment: &hir::PathSegment<'_>,
     ) -> SubstsRef<'tcx> {
-        let (substs, assoc_bindings, _) = self.create_substs_for_ast_path(
+        let (substs, _) = self.create_substs_for_ast_path(
             span,
             def_id,
             &[],
@@ -272,6 +281,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             item_segment.infer_args,
             None,
         );
+        let assoc_bindings = self.create_assoc_bindings_for_generic_args(item_segment.args());
 
         if let Some(b) = assoc_bindings.first() {
             Self::prohibit_assoc_ty_binding(self.tcx(), b.span);
@@ -311,6 +321,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     /// `[Vec<u8>, u8]` and `generic_args` are the arguments for the associated
     /// type itself: `['a]`. The returned `SubstsRef` concatenates these two
     /// lists: `[Vec<u8>, u8, 'a]`.
+    #[tracing::instrument(level = "debug", skip(self, span))]
     fn create_substs_for_ast_path<'a>(
         &self,
         span: Span,
@@ -320,15 +331,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         generic_args: &'a hir::GenericArgs<'_>,
         infer_args: bool,
         self_ty: Option<Ty<'tcx>>,
-    ) -> (SubstsRef<'tcx>, Vec<ConvertedBinding<'a, 'tcx>>, GenericArgCountResult) {
+    ) -> (SubstsRef<'tcx>, GenericArgCountResult) {
         // If the type is parameterized by this region, then replace this
         // region with the current anon region binding (in other words,
         // whatever & would get replaced with).
-        debug!(
-            "create_substs_for_ast_path(def_id={:?}, self_ty={:?}, \
-                generic_args={:?})",
-            def_id, self_ty, generic_args
-        );
 
         let tcx = self.tcx();
         let generics = tcx.generics_of(def_id);
@@ -364,7 +370,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         // here and so associated type bindings will be handled regardless of whether there are any
         // non-`Self` generic parameters.
         if generics.params.len() == 0 {
-            return (tcx.intern_substs(&[]), vec![], arg_count);
+            return (tcx.intern_substs(&[]), arg_count);
         }
 
         let is_object = self_ty.map_or(false, |ty| ty == self.tcx().types.trait_object_dummy_self);
@@ -443,7 +449,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             self.astconv.ast_ty_to_ty(&ty).into()
                         }
                     }
-                    (GenericParamDefKind::Const, GenericArg::Const(ct)) => {
+                    (GenericParamDefKind::Const { .. }, GenericArg::Const(ct)) => {
                         ty::Const::from_opt_const_arg_anon_const(
                             tcx,
                             ty::WithOptConstParam {
@@ -504,15 +510,17 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             tcx.ty_error().into()
                         }
                     }
-                    GenericParamDefKind::Const => {
+                    GenericParamDefKind::Const { has_default } => {
                         let ty = tcx.at(self.span).type_of(param.def_id);
-                        // FIXME(const_generics_defaults)
-                        if infer_args {
-                            // No const parameters were provided, we can infer all.
-                            self.astconv.ct_infer(ty, Some(param), self.span).into()
+                        if !infer_args && has_default {
+                            tcx.const_param_default(param.def_id).into()
                         } else {
-                            // We've already errored above about the mismatch.
-                            tcx.const_error(ty).into()
+                            if infer_args {
+                                self.astconv.ct_infer(ty, Some(param), self.span).into()
+                            } else {
+                                // We've already errored above about the mismatch.
+                                tcx.const_error(ty).into()
+                            }
                         }
                     }
                 }
@@ -535,7 +543,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             parent_substs,
             self_ty.is_some(),
             self_ty,
-            arg_count.clone(),
+            &arg_count,
             &mut substs_ctx,
         );
 
@@ -546,6 +554,18 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             generic_args.args.is_empty(),
         );
 
+        debug!(
+            "create_substs_for_ast_path(generic_params={:?}, self_ty={:?}) -> {:?}",
+            generics, self_ty, substs
+        );
+
+        (substs, arg_count)
+    }
+
+    fn create_assoc_bindings_for_generic_args<'a>(
+        &self,
+        generic_args: &'a hir::GenericArgs<'_>,
+    ) -> Vec<ConvertedBinding<'a, 'tcx>> {
         // Convert associated-type bindings or constraints into a separate vector.
         // Example: Given this:
         //
@@ -576,12 +596,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             })
             .collect();
 
-        debug!(
-            "create_substs_for_ast_path(generic_params={:?}, self_ty={:?}) -> {:?}",
-            generics, self_ty, substs
-        );
-
-        (substs, assoc_bindings, arg_count)
+        assoc_bindings
     }
 
     crate fn create_substs_for_associated_item(
@@ -631,8 +646,27 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         )
     }
 
-    /// The given trait-ref must actually be a trait.
-    pub(super) fn instantiate_poly_trait_ref_inner(
+    /// Given a trait bound like `Debug`, applies that trait bound the given self-type to construct
+    /// a full trait reference. The resulting trait reference is returned. This may also generate
+    /// auxiliary bounds, which are added to `bounds`.
+    ///
+    /// Example:
+    ///
+    /// ```
+    /// poly_trait_ref = Iterator<Item = u32>
+    /// self_ty = Foo
+    /// ```
+    ///
+    /// this would return `Foo: Iterator` and add `<Foo as Iterator>::Item = u32` into `bounds`.
+    ///
+    /// **A note on binders:** against our usual convention, there is an implied bounder around
+    /// the `self_ty` and `poly_trait_ref` parameters here. So they may reference bound regions.
+    /// If for example you had `for<'a> Foo<'a>: Bar<'a>`, then the `self_ty` would be `Foo<'a>`
+    /// where `'a` is a bound region at depth 0. Similarly, the `poly_trait_ref` would be
+    /// `Bar<'a>`. The returned poly-trait-ref will have this binder instantiated explicitly,
+    /// however.
+    #[tracing::instrument(level = "debug", skip(self, span, constness, bounds, speculative))]
+    pub fn instantiate_poly_trait_ref(
         &self,
         trait_ref: &hir::TraitRef<'_>,
         span: Span,
@@ -643,18 +677,25 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     ) -> GenericArgCountResult {
         let trait_def_id = trait_ref.trait_def_id().unwrap_or_else(|| FatalError.raise());
 
-        debug!("instantiate_poly_trait_ref({:?}, def_id={:?})", trait_ref, trait_def_id);
-
         self.prohibit_generics(trait_ref.path.segments.split_last().unwrap().1);
 
-        let (substs, assoc_bindings, arg_count) = self.create_substs_for_ast_trait_ref(
+        let tcx = self.tcx();
+        let bound_vars = tcx.late_bound_vars(trait_ref.hir_ref_id);
+        debug!(?bound_vars);
+
+        let (substs, arg_count) = self.create_substs_for_ast_trait_ref(
             trait_ref.path.span,
             trait_def_id,
             self_ty,
             trait_ref.path.segments.last().unwrap(),
         );
-        let poly_trait_ref = ty::Binder::bind(ty::TraitRef::new(trait_def_id, substs));
+        let assoc_bindings = self
+            .create_assoc_bindings_for_generic_args(trait_ref.path.segments.last().unwrap().args());
+
+        let poly_trait_ref =
+            ty::Binder::bind_with_vars(ty::TraitRef::new(trait_def_id, substs), bound_vars);
 
+        debug!(?poly_trait_ref, ?assoc_bindings);
         bounds.trait_bounds.push((poly_trait_ref, span, constness));
 
         let mut dup_bindings = FxHashMap::default();
@@ -672,50 +713,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             // Okay to ignore `Err` because of `ErrorReported` (see above).
         }
 
-        debug!(
-            "instantiate_poly_trait_ref({:?}, bounds={:?}) -> {:?}",
-            trait_ref, bounds, poly_trait_ref
-        );
-
         arg_count
     }
 
-    /// Given a trait bound like `Debug`, applies that trait bound the given self-type to construct
-    /// a full trait reference. The resulting trait reference is returned. This may also generate
-    /// auxiliary bounds, which are added to `bounds`.
-    ///
-    /// Example:
-    ///
-    /// ```
-    /// poly_trait_ref = Iterator<Item = u32>
-    /// self_ty = Foo
-    /// ```
-    ///
-    /// this would return `Foo: Iterator` and add `<Foo as Iterator>::Item = u32` into `bounds`.
-    ///
-    /// **A note on binders:** against our usual convention, there is an implied bounder around
-    /// the `self_ty` and `poly_trait_ref` parameters here. So they may reference bound regions.
-    /// If for example you had `for<'a> Foo<'a>: Bar<'a>`, then the `self_ty` would be `Foo<'a>`
-    /// where `'a` is a bound region at depth 0. Similarly, the `poly_trait_ref` would be
-    /// `Bar<'a>`. The returned poly-trait-ref will have this binder instantiated explicitly,
-    /// however.
-    pub fn instantiate_poly_trait_ref(
-        &self,
-        poly_trait_ref: &hir::PolyTraitRef<'_>,
-        constness: Constness,
-        self_ty: Ty<'tcx>,
-        bounds: &mut Bounds<'tcx>,
-    ) -> GenericArgCountResult {
-        self.instantiate_poly_trait_ref_inner(
-            &poly_trait_ref.trait_ref,
-            poly_trait_ref.span,
-            constness,
-            self_ty,
-            bounds,
-            false,
-        )
-    }
-
     pub fn instantiate_lang_item_trait_ref(
         &self,
         lang_item: hir::LangItem,
@@ -727,7 +727,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     ) {
         let trait_def_id = self.tcx().require_lang_item(lang_item, Some(span));
 
-        let (substs, assoc_bindings, _) = self.create_substs_for_ast_path(
+        let (substs, _) = self.create_substs_for_ast_path(
             span,
             trait_def_id,
             &[],
@@ -736,7 +736,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             false,
             Some(self_ty),
         );
-        let poly_trait_ref = ty::Binder::bind(ty::TraitRef::new(trait_def_id, substs));
+        let assoc_bindings = self.create_assoc_bindings_for_generic_args(args);
+        let tcx = self.tcx();
+        let bound_vars = tcx.late_bound_vars(hir_id);
+        let poly_trait_ref =
+            ty::Binder::bind_with_vars(ty::TraitRef::new(trait_def_id, substs), bound_vars);
         bounds.trait_bounds.push((poly_trait_ref, span, Constness::NotConst));
 
         let mut dup_bindings = FxHashMap::default();
@@ -760,23 +764,23 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         self_ty: Ty<'tcx>,
         trait_segment: &hir::PathSegment<'_>,
     ) -> ty::TraitRef<'tcx> {
-        let (substs, assoc_bindings, _) =
+        let (substs, _) =
             self.create_substs_for_ast_trait_ref(span, trait_def_id, self_ty, trait_segment);
+        let assoc_bindings = self.create_assoc_bindings_for_generic_args(trait_segment.args());
         if let Some(b) = assoc_bindings.first() {
             Self::prohibit_assoc_ty_binding(self.tcx(), b.span);
         }
         ty::TraitRef::new(trait_def_id, substs)
     }
 
+    #[tracing::instrument(level = "debug", skip(self, span))]
     fn create_substs_for_ast_trait_ref<'a>(
         &self,
         span: Span,
         trait_def_id: DefId,
         self_ty: Ty<'tcx>,
         trait_segment: &'a hir::PathSegment<'a>,
-    ) -> (SubstsRef<'tcx>, Vec<ConvertedBinding<'a, 'tcx>>, GenericArgCountResult) {
-        debug!("create_substs_for_ast_trait_ref(trait_segment={:?})", trait_segment);
-
+    ) -> (SubstsRef<'tcx>, GenericArgCountResult) {
         self.complain_about_internal_fn_trait(span, trait_def_id, trait_segment);
 
         self.create_substs_for_ast_path(
@@ -798,7 +802,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     }
 
     // Returns `true` if a bounds list includes `?Sized`.
-    pub fn is_unsized(&self, ast_bounds: &[&hir::GenericBound<'_>], span: Span) -> bool {
+    pub fn is_unsized(&self, ast_bounds: &[hir::GenericBound<'_>], span: Span) -> bool {
         let tcx = self.tcx();
 
         // Try to find an unbound in bounds.
@@ -853,29 +857,46 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     /// **A note on binders:** there is an implied binder around
     /// `param_ty` and `ast_bounds`. See `instantiate_poly_trait_ref`
     /// for more details.
+    #[tracing::instrument(level = "debug", skip(self, bounds))]
     fn add_bounds(
         &self,
         param_ty: Ty<'tcx>,
-        ast_bounds: &[&hir::GenericBound<'_>],
+        ast_bounds: &[hir::GenericBound<'_>],
         bounds: &mut Bounds<'tcx>,
+        bound_vars: &'tcx ty::List<ty::BoundVariableKind>,
     ) {
         let constness = self.default_constness_for_trait_bounds();
         for ast_bound in ast_bounds {
             match *ast_bound {
                 hir::GenericBound::Trait(ref b, hir::TraitBoundModifier::None) => {
-                    self.instantiate_poly_trait_ref(b, constness, param_ty, bounds);
+                    self.instantiate_poly_trait_ref(
+                        &b.trait_ref,
+                        b.span,
+                        constness,
+                        param_ty,
+                        bounds,
+                        false,
+                    );
                 }
                 hir::GenericBound::Trait(ref b, hir::TraitBoundModifier::MaybeConst) => {
-                    self.instantiate_poly_trait_ref(b, Constness::NotConst, param_ty, bounds);
+                    self.instantiate_poly_trait_ref(
+                        &b.trait_ref,
+                        b.span,
+                        Constness::NotConst,
+                        param_ty,
+                        bounds,
+                        false,
+                    );
                 }
                 hir::GenericBound::Trait(_, hir::TraitBoundModifier::Maybe) => {}
                 hir::GenericBound::LangItemTrait(lang_item, span, hir_id, args) => self
                     .instantiate_lang_item_trait_ref(
-                        *lang_item, *span, *hir_id, args, param_ty, bounds,
+                        lang_item, span, hir_id, args, param_ty, bounds,
                     ),
-                hir::GenericBound::Outlives(ref l) => bounds
-                    .region_bounds
-                    .push((ty::Binder::bind(self.ast_region_to_region(l, None)), l.span)),
+                hir::GenericBound::Outlives(ref l) => bounds.region_bounds.push((
+                    ty::Binder::bind_with_vars(self.ast_region_to_region(l, None), bound_vars),
+                    l.span,
+                )),
             }
         }
     }
@@ -903,7 +924,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         sized_by_default: SizedByDefault,
         span: Span,
     ) -> Bounds<'tcx> {
-        let ast_bounds: Vec<_> = ast_bounds.iter().collect();
         self.compute_bounds_inner(param_ty, &ast_bounds, sized_by_default, span)
     }
 
@@ -923,7 +943,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             if let Some(trait_ref) = ast_bound.trait_ref() {
                 if let Some(trait_did) = trait_ref.trait_def_id() {
                     if self.tcx().trait_may_define_assoc_type(trait_did, assoc_name) {
-                        result.push(ast_bound);
+                        result.push(ast_bound.clone());
                     }
                 }
             }
@@ -935,14 +955,13 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     fn compute_bounds_inner(
         &self,
         param_ty: Ty<'tcx>,
-        ast_bounds: &[&hir::GenericBound<'_>],
+        ast_bounds: &[hir::GenericBound<'_>],
         sized_by_default: SizedByDefault,
         span: Span,
     ) -> Bounds<'tcx> {
         let mut bounds = Bounds::default();
 
-        self.add_bounds(param_ty, ast_bounds, &mut bounds);
-        bounds.trait_bounds.sort_by_key(|(t, _, _)| t.def_id());
+        self.add_bounds(param_ty, ast_bounds, &mut bounds, ty::List::empty());
 
         bounds.implicitly_sized = if let SizedByDefault::Yes = sized_by_default {
             if !self.is_unsized(ast_bounds, span) { Some(span) } else { None }
@@ -959,6 +978,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     /// **A note on binders:** given something like `T: for<'a> Iterator<Item = &'a u32>`, the
     /// `trait_ref` here will be `for<'a> T: Iterator`. The `binding` data however is from *inside*
     /// the binder (e.g., `&'a u32`) and hence may reference bound regions.
+    #[tracing::instrument(
+        level = "debug",
+        skip(self, bounds, speculative, dup_bindings, path_span)
+    )]
     fn add_predicates_for_ast_type_binding(
         &self,
         hir_ref_id: hir::HirId,
@@ -985,7 +1008,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         //
         // We want to produce `<B as SuperTrait<i32>>::T == foo`.
 
-        debug!(?hir_ref_id, ?trait_ref, ?binding, ?bounds, "add_predicates_for_ast_type_binding",);
         let tcx = self.tcx();
 
         let candidate =
@@ -1086,7 +1108,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 let late_bound_in_trait_ref =
                     tcx.collect_constrained_late_bound_regions(&projection_ty);
                 let late_bound_in_ty =
-                    tcx.collect_referenced_late_bound_regions(&ty::Binder::bind(ty));
+                    tcx.collect_referenced_late_bound_regions(&trait_ref.rebind(ty));
                 debug!("late_bound_in_trait_ref = {:?}", late_bound_in_trait_ref);
                 debug!("late_bound_in_ty = {:?}", late_bound_in_ty);
 
@@ -1135,10 +1157,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 //
                 // Calling `skip_binder` is okay, because `add_bounds` expects the `param_ty`
                 // parameter to have a skipped binder.
-                let param_ty =
-                    tcx.mk_projection(assoc_ty.def_id, projection_ty.skip_binder().substs);
-                let ast_bounds: Vec<_> = ast_bounds.iter().collect();
-                self.add_bounds(param_ty, &ast_bounds, bounds);
+                let param_ty = tcx.mk_ty(ty::Projection(projection_ty.skip_binder()));
+                self.add_bounds(param_ty, ast_bounds, bounds, candidate.bound_vars());
             }
         }
         Ok(())
@@ -1172,10 +1192,12 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     Err(GenericArgCountMismatch { invalid_args: cur_potential_assoc_types, .. }),
                 ..
             } = self.instantiate_poly_trait_ref(
-                trait_bound,
+                &trait_bound.trait_ref,
+                trait_bound.span,
                 Constness::NotConst,
                 dummy_self,
                 &mut bounds,
+                false,
             ) {
                 potential_assoc_types.extend(cur_potential_assoc_types);
             }
@@ -1318,8 +1340,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         // De-duplicate auto traits so that, e.g., `dyn Trait + Send + Send` is the same as
         // `dyn Trait + Send`.
-        auto_traits.sort_by_key(|i| i.trait_ref().def_id());
-        auto_traits.dedup_by_key(|i| i.trait_ref().def_id());
+        // We remove duplicates by inserting into a `FxHashSet` to avoid re-ordering
+        // the bounds
+        let mut duplicates = FxHashSet::default();
+        auto_traits.retain(|i| duplicates.insert(i.trait_ref().def_id()));
         debug!("regular_traits: {:?}", regular_traits);
         debug!("auto_traits: {:?}", auto_traits);
 
@@ -1413,8 +1437,13 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         name: Symbol,
     ) {
         let mut err = struct_span_err!(self.tcx().sess, span, E0223, "ambiguous associated type");
-        if let (Some(_), Ok(snippet)) = (
-            self.tcx().sess.confused_type_with_std_module.borrow().get(&span),
+        if let (true, Ok(snippet)) = (
+            self.tcx()
+                .sess
+                .confused_type_with_std_module
+                .borrow()
+                .keys()
+                .any(|full_span| full_span.contains(span)),
             self.tcx().sess.source_map().span_to_snippet(span),
         ) {
             err.span_suggestion(
@@ -1602,6 +1631,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     // the whole path.
     // Will fail except for `T::A` and `Self::A`; i.e., if `qself_ty`/`qself_def` are not a type
     // parameter or `Self`.
+    // NOTE: When this function starts resolving `Trait::AssocTy` successfully
+    // it should also start reportint the `BARE_TRAIT_OBJECTS` lint.
     pub fn associated_path_to_ty(
         &self,
         hir_ref_id: hir::HirId,
@@ -1651,7 +1682,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 };
 
                 self.one_bound_for_assoc_type(
-                    || traits::supertraits(tcx, ty::Binder::bind(trait_ref)),
+                    || traits::supertraits(tcx, ty::Binder::bind(trait_ref, tcx)),
                     || "Self".to_string(),
                     assoc_ident,
                     span,
@@ -2163,9 +2194,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
     /// Turns a `hir::Ty` into a `Ty`. For diagnostics' purposes we keep track of whether trait
     /// objects are borrowed like `&dyn Trait` to avoid emitting redundant errors.
+    #[tracing::instrument(level = "debug", skip(self))]
     fn ast_ty_to_ty_inner(&self, ast_ty: &hir::Ty<'_>, borrowed: bool) -> Ty<'tcx> {
-        debug!("ast_ty_to_ty(id={:?}, ast_ty={:?} ty_ty={:?})", ast_ty.hir_id, ast_ty, ast_ty.kind);
-
         let tcx = self.tcx();
 
         let result_ty = match ast_ty.kind {
@@ -2175,7 +2205,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             }
             hir::TyKind::Rptr(ref region, ref mt) => {
                 let r = self.ast_region_to_region(region, None);
-                debug!("ast_ty_to_ty: r={:?}", r);
+                debug!(?r);
                 let t = self.ast_ty_to_ty_inner(&mt.ty, true);
                 tcx.mk_ref(r, ty::TypeAndMut { ty: t, mutbl: mt.mutbl })
             }
@@ -2187,6 +2217,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 require_c_abi_if_c_variadic(tcx, &bf.decl, bf.abi, ast_ty.span);
 
                 tcx.mk_fn_ptr(self.ty_of_fn(
+                    ast_ty.hir_id,
                     bf.unsafety,
                     bf.abi,
                     &bf.decl,
@@ -2195,11 +2226,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     Some(ast_ty),
                 ))
             }
-            hir::TyKind::TraitObject(ref bounds, ref lifetime) => {
+            hir::TyKind::TraitObject(ref bounds, ref lifetime, _) => {
                 self.conv_object_ty_poly_trait_ref(ast_ty.span, bounds, lifetime, borrowed)
             }
             hir::TyKind::Path(hir::QPath::Resolved(ref maybe_qself, ref path)) => {
-                debug!("ast_ty_to_ty: maybe_qself={:?} path={:?}", maybe_qself, path);
+                debug!(?maybe_qself, ?path);
                 let opt_self_ty = maybe_qself.as_ref().map(|qself| self.ast_ty_to_ty(qself));
                 self.res_to_ty(opt_self_ty, path, false)
             }
@@ -2215,7 +2246,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 }
             }
             hir::TyKind::Path(hir::QPath::TypeRelative(ref qself, ref segment)) => {
-                debug!("ast_ty_to_ty: qself={:?} segment={:?}", qself, segment);
+                debug!(?qself, ?segment);
                 let ty = self.ast_ty_to_ty(qself);
 
                 let res = if let hir::TyKind::Path(hir::QPath::Resolved(_, ref path)) = qself.kind {
@@ -2229,7 +2260,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             }
             hir::TyKind::Path(hir::QPath::LangItem(lang_item, span)) => {
                 let def_id = tcx.require_lang_item(lang_item, Some(span));
-                let (substs, _, _) = self.create_substs_for_ast_path(
+                let (substs, _) = self.create_substs_for_ast_path(
                     span,
                     def_id,
                     &[],
@@ -2260,13 +2291,13 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             hir::TyKind::Err => tcx.ty_error(),
         };
 
-        debug!("ast_ty_to_ty: result_ty={:?}", result_ty);
+        debug!(?result_ty);
 
         self.record_ty(ast_ty.hir_id, result_ty, ast_ty.span);
         result_ty
     }
 
-    pub fn impl_trait_ty_to_ty(
+    fn impl_trait_ty_to_ty(
         &self,
         def_id: DefId,
         lifetimes: &[hir::GenericArg<'_>],
@@ -2327,6 +2358,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
     pub fn ty_of_fn(
         &self,
+        hir_id: hir::HirId,
         unsafety: hir::Unsafety,
         abi: abi::Abi,
         decl: &hir::FnDecl<'_>,
@@ -2337,6 +2369,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         debug!("ty_of_fn");
 
         let tcx = self.tcx();
+        let bound_vars = tcx.late_bound_vars(hir_id);
+        debug!(?bound_vars);
 
         // We proactively collect all the inferred type params to emit a single error per fn def.
         let mut visitor = PlaceholderHirTyCollector::default();
@@ -2356,8 +2390,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         debug!("ty_of_fn: output_ty={:?}", output_ty);
 
-        let bare_fn_ty =
-            ty::Binder::bind(tcx.mk_fn_sig(input_tys, output_ty, decl.c_variadic, unsafety, abi));
+        let fn_ty = tcx.mk_fn_sig(input_tys, output_ty, decl.c_variadic, unsafety, abi);
+        let bare_fn_ty = ty::Binder::bind_with_vars(fn_ty, bound_vars);
 
         if !self.allow_ty_infer() {
             // We always collect the spans for placeholder types when evaluating `fn`s, but we
@@ -2438,7 +2472,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     fn compute_object_lifetime_bound(
         &self,
         span: Span,
-        existential_predicates: &'tcx ty::List<ty::Binder<ty::ExistentialPredicate<'tcx>>>,
+        existential_predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
     ) -> Option<ty::Region<'tcx>> // if None, use the default
     {
         let tcx = self.tcx();
diff --git a/compiler/rustc_typeck/src/bounds.rs b/compiler/rustc_typeck/src/bounds.rs
index 7ba90ad8819..5d200640722 100644
--- a/compiler/rustc_typeck/src/bounds.rs
+++ b/compiler/rustc_typeck/src/bounds.rs
@@ -26,7 +26,7 @@ pub struct Bounds<'tcx> {
     /// A list of region bounds on the (implicit) self type. So if you
     /// had `T: 'a + 'b` this might would be a list `['a, 'b]` (but
     /// the `T` is not explicitly included).
-    pub region_bounds: Vec<(ty::Binder<ty::Region<'tcx>>, Span)>,
+    pub region_bounds: Vec<(ty::Binder<'tcx, ty::Region<'tcx>>, Span)>,
 
     /// A list of trait bounds. So if you had `T: Debug` this would be
     /// `T: Debug`. Note that the self-type is explicit here.
@@ -57,7 +57,7 @@ impl<'tcx> Bounds<'tcx> {
         // If it could be sized, and is, add the `Sized` predicate.
         let sized_predicate = self.implicitly_sized.and_then(|span| {
             tcx.lang_items().sized_trait().map(|sized| {
-                let trait_ref = ty::Binder::bind(ty::TraitRef {
+                let trait_ref = ty::Binder::dummy(ty::TraitRef {
                     def_id: sized,
                     substs: tcx.mk_substs_trait(param_ty, &[]),
                 });
diff --git a/compiler/rustc_typeck/src/check/_match.rs b/compiler/rustc_typeck/src/check/_match.rs
index cfd4e0830ef..d056f2c90f9 100644
--- a/compiler/rustc_typeck/src/check/_match.rs
+++ b/compiler/rustc_typeck/src/check/_match.rs
@@ -207,17 +207,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     ),
                 };
                 let cause = self.cause(span, code);
-                let can_coerce_to_return_ty = match self.ret_coercion.as_ref() {
-                    Some(ret_coercion) if self.in_tail_expr => {
-                        let ret_ty = ret_coercion.borrow().expected_ty();
-                        let ret_ty = self.inh.infcx.shallow_resolve(ret_ty);
-                        self.can_coerce(arm_ty, ret_ty)
-                            && prior_arm_ty.map_or(true, |t| self.can_coerce(t, ret_ty))
-                            // The match arms need to unify for the case of `impl Trait`.
-                            && !matches!(ret_ty.kind(), ty::Opaque(..))
-                    }
-                    _ => false,
-                };
 
                 // This is the moral equivalent of `coercion.coerce(self, cause, arm.body, arm_ty)`.
                 // We use it this way to be able to expand on the potential error and detect when a
@@ -229,6 +218,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     Some(&arm.body),
                     arm_ty,
                     Some(&mut |err: &mut DiagnosticBuilder<'_>| {
+                        let can_coerce_to_return_ty = match self.ret_coercion.as_ref() {
+                            Some(ret_coercion) if self.in_tail_expr => {
+                                let ret_ty = ret_coercion.borrow().expected_ty();
+                                let ret_ty = self.inh.infcx.shallow_resolve(ret_ty);
+                                self.can_coerce(arm_ty, ret_ty)
+                                    && prior_arm_ty.map_or(true, |t| self.can_coerce(t, ret_ty))
+                                    // The match arms need to unify for the case of `impl Trait`.
+                                    && !matches!(ret_ty.kind(), ty::Opaque(..))
+                            }
+                            _ => false,
+                        };
                         if let (Expectation::IsLast(stmt), Some(ret), true) =
                             (orig_expected, self.ret_type_span, can_coerce_to_return_ty)
                         {
diff --git a/compiler/rustc_typeck/src/check/callee.rs b/compiler/rustc_typeck/src/check/callee.rs
index ca1e79fac73..b48102e0fc9 100644
--- a/compiler/rustc_typeck/src/check/callee.rs
+++ b/compiler/rustc_typeck/src/check/callee.rs
@@ -17,6 +17,7 @@ use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
 use rustc_target::spec::abi;
 use rustc_trait_selection::autoderef::Autoderef;
+use std::iter;
 
 /// Checks that it is legal to call methods of the trait corresponding
 /// to `trait_id` (this only cares about the trait, not the specific
@@ -77,11 +78,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let output = match result {
             None => {
                 // this will report an error since original_callee_ty is not a fn
-                self.confirm_builtin_call(call_expr, original_callee_ty, arg_exprs, expected)
+                self.confirm_builtin_call(
+                    call_expr,
+                    callee_expr,
+                    original_callee_ty,
+                    arg_exprs,
+                    expected,
+                )
             }
 
             Some(CallStep::Builtin(callee_ty)) => {
-                self.confirm_builtin_call(call_expr, callee_ty, arg_exprs, expected)
+                self.confirm_builtin_call(call_expr, callee_expr, callee_ty, arg_exprs, expected)
             }
 
             Some(CallStep::DeferredClosure(fn_sig)) => {
@@ -281,6 +288,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn confirm_builtin_call(
         &self,
         call_expr: &'tcx hir::Expr<'tcx>,
+        callee_expr: &'tcx hir::Expr<'tcx>,
         callee_ty: Ty<'tcx>,
         arg_exprs: &'tcx [hir::Expr<'tcx>],
         expected: Expectation<'tcx>,
@@ -299,110 +307,104 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
                 }
 
-                if let hir::ExprKind::Call(callee, _) = call_expr.kind {
-                    let mut err = type_error_struct!(
-                        self.tcx.sess,
-                        callee.span,
-                        callee_ty,
-                        E0618,
-                        "expected function, found {}",
-                        match unit_variant {
-                            Some(ref path) => format!("enum variant `{}`", path),
-                            None => format!("`{}`", callee_ty),
-                        }
-                    );
+                let mut err = type_error_struct!(
+                    self.tcx.sess,
+                    callee_expr.span,
+                    callee_ty,
+                    E0618,
+                    "expected function, found {}",
+                    match unit_variant {
+                        Some(ref path) => format!("enum variant `{}`", path),
+                        None => format!("`{}`", callee_ty),
+                    }
+                );
 
-                    self.identify_bad_closure_def_and_call(
-                        &mut err,
-                        call_expr.hir_id,
-                        &callee.kind,
-                        callee.span,
-                    );
+                self.identify_bad_closure_def_and_call(
+                    &mut err,
+                    call_expr.hir_id,
+                    &callee_expr.kind,
+                    callee_expr.span,
+                );
 
-                    if let Some(ref path) = unit_variant {
-                        err.span_suggestion(
-                            call_expr.span,
-                            &format!(
-                                "`{}` is a unit variant, you need to write it \
+                if let Some(ref path) = unit_variant {
+                    err.span_suggestion(
+                        call_expr.span,
+                        &format!(
+                            "`{}` is a unit variant, you need to write it \
                                  without the parenthesis",
-                                path
-                            ),
-                            path.to_string(),
-                            Applicability::MachineApplicable,
-                        );
-                    }
+                            path
+                        ),
+                        path.to_string(),
+                        Applicability::MachineApplicable,
+                    );
+                }
 
-                    let mut inner_callee_path = None;
-                    let def = match callee.kind {
-                        hir::ExprKind::Path(ref qpath) => {
-                            self.typeck_results.borrow().qpath_res(qpath, callee.hir_id)
+                let mut inner_callee_path = None;
+                let def = match callee_expr.kind {
+                    hir::ExprKind::Path(ref qpath) => {
+                        self.typeck_results.borrow().qpath_res(qpath, callee_expr.hir_id)
+                    }
+                    hir::ExprKind::Call(ref inner_callee, _) => {
+                        // If the call spans more than one line and the callee kind is
+                        // itself another `ExprCall`, that's a clue that we might just be
+                        // missing a semicolon (Issue #51055)
+                        let call_is_multiline =
+                            self.tcx.sess.source_map().is_multiline(call_expr.span);
+                        if call_is_multiline {
+                            err.span_suggestion(
+                                callee_expr.span.shrink_to_hi(),
+                                "consider using a semicolon here",
+                                ";".to_owned(),
+                                Applicability::MaybeIncorrect,
+                            );
                         }
-                        hir::ExprKind::Call(ref inner_callee, _) => {
-                            // If the call spans more than one line and the callee kind is
-                            // itself another `ExprCall`, that's a clue that we might just be
-                            // missing a semicolon (Issue #51055)
-                            let call_is_multiline =
-                                self.tcx.sess.source_map().is_multiline(call_expr.span);
-                            if call_is_multiline {
-                                err.span_suggestion(
-                                    callee.span.shrink_to_hi(),
-                                    "consider using a semicolon here",
-                                    ";".to_owned(),
-                                    Applicability::MaybeIncorrect,
-                                );
-                            }
-                            if let hir::ExprKind::Path(ref inner_qpath) = inner_callee.kind {
-                                inner_callee_path = Some(inner_qpath);
-                                self.typeck_results
-                                    .borrow()
-                                    .qpath_res(inner_qpath, inner_callee.hir_id)
-                            } else {
-                                Res::Err
-                            }
+                        if let hir::ExprKind::Path(ref inner_qpath) = inner_callee.kind {
+                            inner_callee_path = Some(inner_qpath);
+                            self.typeck_results.borrow().qpath_res(inner_qpath, inner_callee.hir_id)
+                        } else {
+                            Res::Err
                         }
-                        _ => Res::Err,
-                    };
-
-                    err.span_label(call_expr.span, "call expression requires function");
-
-                    if let Some(span) = self.tcx.hir().res_span(def) {
-                        let callee_ty = callee_ty.to_string();
-                        let label = match (unit_variant, inner_callee_path) {
-                            (Some(path), _) => Some(format!("`{}` defined here", path)),
-                            (_, Some(hir::QPath::Resolved(_, path))) => {
-                                self.tcx.sess.source_map().span_to_snippet(path.span).ok().map(
-                                    |p| format!("`{}` defined here returns `{}`", p, callee_ty),
-                                )
-                            }
-                            _ => {
-                                match def {
-                                    // Emit a different diagnostic for local variables, as they are not
-                                    // type definitions themselves, but rather variables *of* that type.
-                                    Res::Local(hir_id) => Some(format!(
-                                        "`{}` has type `{}`",
-                                        self.tcx.hir().name(hir_id),
-                                        callee_ty
-                                    )),
-                                    Res::Def(kind, def_id)
-                                        if kind.ns() == Some(Namespace::ValueNS) =>
-                                    {
-                                        Some(format!(
-                                            "`{}` defined here",
-                                            self.tcx.def_path_str(def_id),
-                                        ))
-                                    }
-                                    _ => Some(format!("`{}` defined here", callee_ty)),
+                    }
+                    _ => Res::Err,
+                };
+
+                err.span_label(call_expr.span, "call expression requires function");
+
+                if let Some(span) = self.tcx.hir().res_span(def) {
+                    let callee_ty = callee_ty.to_string();
+                    let label = match (unit_variant, inner_callee_path) {
+                        (Some(path), _) => Some(format!("`{}` defined here", path)),
+                        (_, Some(hir::QPath::Resolved(_, path))) => self
+                            .tcx
+                            .sess
+                            .source_map()
+                            .span_to_snippet(path.span)
+                            .ok()
+                            .map(|p| format!("`{}` defined here returns `{}`", p, callee_ty)),
+                        _ => {
+                            match def {
+                                // Emit a different diagnostic for local variables, as they are not
+                                // type definitions themselves, but rather variables *of* that type.
+                                Res::Local(hir_id) => Some(format!(
+                                    "`{}` has type `{}`",
+                                    self.tcx.hir().name(hir_id),
+                                    callee_ty
+                                )),
+                                Res::Def(kind, def_id) if kind.ns() == Some(Namespace::ValueNS) => {
+                                    Some(format!(
+                                        "`{}` defined here",
+                                        self.tcx.def_path_str(def_id),
+                                    ))
                                 }
+                                _ => Some(format!("`{}` defined here", callee_ty)),
                             }
-                        };
-                        if let Some(label) = label {
-                            err.span_label(span, label);
                         }
+                    };
+                    if let Some(label) = label {
+                        err.span_label(span, label);
                     }
-                    err.emit();
-                } else {
-                    bug!("call_expr.kind should be an ExprKind::Call, got {:?}", call_expr.kind);
                 }
+                err.emit();
 
                 // This is the "default" function signature, used in case of error.
                 // In that case, we check each argument against "error" in order to
@@ -465,7 +467,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let expected_arg_tys = self.expected_inputs_for_expected_output(
             call_expr.span,
             expected,
-            fn_sig.output().clone(),
+            fn_sig.output(),
             fn_sig.inputs(),
         );
 
@@ -538,7 +540,7 @@ impl<'a, 'tcx> DeferredCallResolution<'tcx> {
                 debug!("attempt_resolution: method_callee={:?}", method_callee);
 
                 for (method_arg_ty, self_arg_ty) in
-                    method_sig.inputs().iter().skip(1).zip(self.fn_sig.inputs())
+                    iter::zip(method_sig.inputs().iter().skip(1), self.fn_sig.inputs())
                 {
                     fcx.demand_eqtype(self.call_expr.span, &self_arg_ty, &method_arg_ty);
                 }
diff --git a/compiler/rustc_typeck/src/check/cast.rs b/compiler/rustc_typeck/src/check/cast.rs
index 16c344e8e2b..b760a54f08c 100644
--- a/compiler/rustc_typeck/src/check/cast.rs
+++ b/compiler/rustc_typeck/src/check/cast.rs
@@ -359,6 +359,21 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                     {
                         sugg = Some(format!("&{}", mutbl.prefix_str()));
                     }
+                } else if let ty::RawPtr(TypeAndMut { mutbl, .. }) = *self.cast_ty.kind() {
+                    if fcx
+                        .try_coerce(
+                            self.expr,
+                            fcx.tcx.mk_ref(
+                                &ty::RegionKind::ReErased,
+                                TypeAndMut { ty: self.expr_ty, mutbl },
+                            ),
+                            self.cast_ty,
+                            AllowTwoPhase::No,
+                        )
+                        .is_ok()
+                    {
+                        sugg = Some(format!("&{}", mutbl.prefix_str()));
+                    }
                 }
                 if let Some(sugg) = sugg {
                     err.span_label(self.span, "invalid cast");
diff --git a/compiler/rustc_typeck/src/check/check.rs b/compiler/rustc_typeck/src/check/check.rs
index 0010d59f710..892abb5a344 100644
--- a/compiler/rustc_typeck/src/check/check.rs
+++ b/compiler/rustc_typeck/src/check/check.rs
@@ -26,6 +26,7 @@ use rustc_trait_selection::opaque_types::InferCtxtExt as _;
 use rustc_trait_selection::traits::error_reporting::InferCtxtExt as _;
 use rustc_trait_selection::traits::{self, ObligationCauseCode};
 
+use std::iter;
 use std::ops::ControlFlow;
 
 pub fn check_wf_new(tcx: TyCtxt<'_>) {
@@ -87,8 +88,69 @@ pub(super) fn check_fn<'a, 'tcx>(
 
     let declared_ret_ty = fn_sig.output();
 
-    let revealed_ret_ty =
-        fcx.instantiate_opaque_types_from_value(fn_id, declared_ret_ty, decl.output.span());
+    let feature = match tcx.hir().get(fn_id) {
+        // TAIT usage in function return position.
+        // Example:
+        //
+        // ```rust
+        // type Foo = impl Debug;
+        // fn bar() -> Foo { 42 }
+        // ```
+        Node::Item(hir::Item { kind: ItemKind::Fn(..), .. }) |
+        // TAIT usage in associated function return position.
+        //
+        // Example with a free type alias:
+        //
+        // ```rust
+        // type Foo = impl Debug;
+        // impl SomeTrait for SomeType {
+        //     fn bar() -> Foo { 42 }
+        // }
+        // ```
+        //
+        // Example with an associated TAIT:
+        //
+        // ```rust
+        // impl SomeTrait for SomeType {
+        //     type Foo = impl Debug;
+        //     fn bar() -> Self::Foo { 42 }
+        // }
+        // ```
+        Node::ImplItem(hir::ImplItem {
+            kind: hir::ImplItemKind::Fn(..), ..
+        }) => None,
+        // Forbid TAIT in trait declarations for now.
+        // Examples:
+        //
+        // ```rust
+        // type Foo = impl Debug;
+        // trait Bar {
+        //     fn bar() -> Foo;
+        // }
+        // trait Bop {
+        //     type Bop: PartialEq<Foo>;
+        // }
+        // ```
+        Node::TraitItem(hir::TraitItem {
+            kind: hir::TraitItemKind::Fn(..),
+            ..
+        }) |
+        // Forbid TAIT in closure return position for now.
+        // Example:
+        //
+        // ```rust
+        // type Foo = impl Debug;
+        // let x = |y| -> Foo { 42 + y };
+        // ```
+        Node::Expr(hir::Expr { kind: hir::ExprKind::Closure(..), .. }) => Some(sym::type_alias_impl_trait),
+        node => bug!("Item being checked wasn't a function/closure: {:?}", node),
+    };
+    let revealed_ret_ty = fcx.instantiate_opaque_types_from_value(
+        fn_id,
+        declared_ret_ty,
+        decl.output.span(),
+        feature,
+    );
     debug!("check_fn: declared_ret_ty: {}, revealed_ret_ty: {}", declared_ret_ty, revealed_ret_ty);
     fcx.ret_coercion = Some(RefCell::new(CoerceMany::new(revealed_ret_ty)));
     fcx.ret_type_span = Some(decl.output.span());
@@ -659,7 +721,8 @@ fn check_opaque_meets_bounds<'tcx>(
         // Checked when type checking the function containing them.
         hir::OpaqueTyOrigin::FnReturn | hir::OpaqueTyOrigin::AsyncFn => return,
         // Can have different predicates to their defining use
-        hir::OpaqueTyOrigin::Binding | hir::OpaqueTyOrigin::Misc => {}
+        hir::OpaqueTyOrigin::Binding | hir::OpaqueTyOrigin::Misc | hir::OpaqueTyOrigin::TyAlias => {
+        }
     }
 
     let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
@@ -1410,7 +1473,7 @@ fn check_enum<'tcx>(
     }
 
     let mut disr_vals: Vec<Discr<'tcx>> = Vec::with_capacity(vs.len());
-    for ((_, discr), v) in def.discriminants(tcx).zip(vs) {
+    for ((_, discr), v) in iter::zip(def.discriminants(tcx), vs) {
         // Check for duplicate discriminant values
         if let Some(i) = disr_vals.iter().position(|&x| x.val == discr.val) {
             let variant_did = def.variants[VariantIdx::new(i)].def_id;
diff --git a/compiler/rustc_typeck/src/check/closure.rs b/compiler/rustc_typeck/src/check/closure.rs
index 431e6d70ff3..22d3dc6bdc0 100644
--- a/compiler/rustc_typeck/src/check/closure.rs
+++ b/compiler/rustc_typeck/src/check/closure.rs
@@ -69,7 +69,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let expr_def_id = self.tcx.hir().local_def_id(expr.hir_id);
 
         let ClosureSignatures { bound_sig, liberated_sig } =
-            self.sig_of_closure(expr_def_id.to_def_id(), decl, body, expected_sig);
+            self.sig_of_closure(expr.hir_id, expr_def_id.to_def_id(), decl, body, expected_sig);
 
         debug!("check_closure: ty_of_closure returns {:?}", liberated_sig);
 
@@ -288,15 +288,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     fn sig_of_closure(
         &self,
+        hir_id: hir::HirId,
         expr_def_id: DefId,
         decl: &hir::FnDecl<'_>,
         body: &hir::Body<'_>,
         expected_sig: Option<ExpectedSig<'tcx>>,
     ) -> ClosureSignatures<'tcx> {
         if let Some(e) = expected_sig {
-            self.sig_of_closure_with_expectation(expr_def_id, decl, body, e)
+            self.sig_of_closure_with_expectation(hir_id, expr_def_id, decl, body, e)
         } else {
-            self.sig_of_closure_no_expectation(expr_def_id, decl, body)
+            self.sig_of_closure_no_expectation(hir_id, expr_def_id, decl, body)
         }
     }
 
@@ -304,13 +305,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// types that the user gave into a signature.
     fn sig_of_closure_no_expectation(
         &self,
+        hir_id: hir::HirId,
         expr_def_id: DefId,
         decl: &hir::FnDecl<'_>,
         body: &hir::Body<'_>,
     ) -> ClosureSignatures<'tcx> {
         debug!("sig_of_closure_no_expectation()");
 
-        let bound_sig = self.supplied_sig_of_closure(expr_def_id, decl, body);
+        let bound_sig = self.supplied_sig_of_closure(hir_id, expr_def_id, decl, body);
 
         self.closure_sigs(expr_def_id, body, bound_sig)
     }
@@ -364,6 +366,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ///   regions with depth 1, which are bound then by the closure.
     fn sig_of_closure_with_expectation(
         &self,
+        hir_id: hir::HirId,
         expr_def_id: DefId,
         decl: &hir::FnDecl<'_>,
         body: &hir::Body<'_>,
@@ -375,7 +378,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // expectation if things don't see to match up with what we
         // expect.
         if expected_sig.sig.c_variadic() != decl.c_variadic {
-            return self.sig_of_closure_no_expectation(expr_def_id, decl, body);
+            return self.sig_of_closure_no_expectation(hir_id, expr_def_id, decl, body);
         } else if expected_sig.sig.skip_binder().inputs_and_output.len() != decl.inputs.len() + 1 {
             return self.sig_of_closure_with_mismatched_number_of_arguments(
                 expr_def_id,
@@ -411,9 +414,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Along the way, it also writes out entries for types that the user
         // wrote into our typeck results, which are then later used by the privacy
         // check.
-        match self.check_supplied_sig_against_expectation(expr_def_id, decl, body, &closure_sigs) {
+        match self.check_supplied_sig_against_expectation(
+            hir_id,
+            expr_def_id,
+            decl,
+            body,
+            &closure_sigs,
+        ) {
             Ok(infer_ok) => self.register_infer_ok_obligations(infer_ok),
-            Err(_) => return self.sig_of_closure_no_expectation(expr_def_id, decl, body),
+            Err(_) => return self.sig_of_closure_no_expectation(hir_id, expr_def_id, decl, body),
         }
 
         closure_sigs
@@ -460,6 +469,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// strategy.
     fn check_supplied_sig_against_expectation(
         &self,
+        hir_id: hir::HirId,
         expr_def_id: DefId,
         decl: &hir::FnDecl<'_>,
         body: &hir::Body<'_>,
@@ -469,7 +479,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         //
         // (See comment on `sig_of_closure_with_expectation` for the
         // meaning of these letters.)
-        let supplied_sig = self.supplied_sig_of_closure(expr_def_id, decl, body);
+        let supplied_sig = self.supplied_sig_of_closure(hir_id, expr_def_id, decl, body);
 
         debug!("check_supplied_sig_against_expectation: supplied_sig={:?}", supplied_sig);
 
@@ -492,13 +502,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             // The liberated version of this signature should be a subtype
             // of the liberated form of the expectation.
-            for ((hir_ty, &supplied_ty), expected_ty) in decl
-                .inputs
-                .iter()
-                .zip(supplied_sig.inputs().skip_binder()) // binder moved to (*) below
-                .zip(expected_sigs.liberated_sig.inputs())
-            // `liberated_sig` is E'.
-            {
+            for ((hir_ty, &supplied_ty), expected_ty) in iter::zip(
+                iter::zip(
+                    decl.inputs,
+                    supplied_sig.inputs().skip_binder(), // binder moved to (*) below
+                ),
+                expected_sigs.liberated_sig.inputs(), // `liberated_sig` is E'.
+            ) {
                 // Instantiate (this part of..) S to S', i.e., with fresh variables.
                 let (supplied_ty, _) = self.infcx.replace_bound_vars_with_fresh_vars(
                     hir_ty.span,
@@ -534,6 +544,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// Also, record this closure signature for later.
     fn supplied_sig_of_closure(
         &self,
+        hir_id: hir::HirId,
         expr_def_id: DefId,
         decl: &hir::FnDecl<'_>,
         body: &hir::Body<'_>,
@@ -545,6 +556,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             decl, body.generator_kind,
         );
 
+        let bound_vars = self.tcx.late_bound_vars(hir_id);
+
         // First, convert the types that the user supplied (if any).
         let supplied_arguments = decl.inputs.iter().map(|a| astconv.ast_ty_to_ty(a));
         let supplied_return = match decl.output {
@@ -571,13 +584,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             },
         };
 
-        let result = ty::Binder::bind(self.tcx.mk_fn_sig(
-            supplied_arguments,
-            supplied_return,
-            decl.c_variadic,
-            hir::Unsafety::Normal,
-            Abi::RustCall,
-        ));
+        let result = ty::Binder::bind_with_vars(
+            self.tcx.mk_fn_sig(
+                supplied_arguments,
+                supplied_return,
+                decl.c_variadic,
+                hir::Unsafety::Normal,
+                Abi::RustCall,
+            ),
+            bound_vars,
+        );
 
         debug!("supplied_sig_of_closure: result={:?}", result);
 
diff --git a/compiler/rustc_typeck/src/check/coercion.rs b/compiler/rustc_typeck/src/check/coercion.rs
index 32b3d0b0595..427f967a9b6 100644
--- a/compiler/rustc_typeck/src/check/coercion.rs
+++ b/compiler/rustc_typeck/src/check/coercion.rs
@@ -973,6 +973,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         };
         if let (Some(a_sig), Some(b_sig)) = (a_sig, b_sig) {
+            // Intrinsics are not coercible to function pointers.
+            if a_sig.abi() == Abi::RustIntrinsic
+                || a_sig.abi() == Abi::PlatformIntrinsic
+                || b_sig.abi() == Abi::RustIntrinsic
+                || b_sig.abi() == Abi::PlatformIntrinsic
+            {
+                return Err(TypeError::IntrinsicCast);
+            }
             // The signature must match.
             let a_sig = self.normalize_associated_types_in(new.span, a_sig);
             let b_sig = self.normalize_associated_types_in(new.span, b_sig);
@@ -1440,9 +1448,8 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         // as prior return coercions would not be relevant (#57664).
         let parent_id = fcx.tcx.hir().get_parent_node(id);
         let fn_decl = if let Some((expr, blk_id)) = expression {
-            pointing_at_return_type = fcx.suggest_mismatched_types_on_tail(
-                &mut err, expr, expected, found, cause.span, blk_id,
-            );
+            pointing_at_return_type =
+                fcx.suggest_mismatched_types_on_tail(&mut err, expr, expected, found, blk_id);
             let parent = fcx.tcx.hir().get(parent_id);
             if let (Some(cond_expr), true, false) = (
                 fcx.tcx.hir().get_if_cause(expr.hir_id),
@@ -1487,34 +1494,12 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         if let (Some((expr, _)), Some((fn_decl, _, _))) =
             (expression, fcx.get_node_fn_decl(parent_item))
         {
-            fcx.suggest_missing_return_expr(&mut err, expr, fn_decl, expected, found);
+            fcx.suggest_missing_return_expr(&mut err, expr, fn_decl, expected, found, parent_id);
         }
 
         if let (Some(sp), Some(fn_output)) = (fcx.ret_coercion_span.get(), fn_output) {
             self.add_impl_trait_explanation(&mut err, cause, fcx, expected, sp, fn_output);
         }
-
-        if let Some(sp) = fcx.ret_coercion_span.get() {
-            // If the closure has an explicit return type annotation,
-            // then a type error may occur at the first return expression we
-            // see in the closure (if it conflicts with the declared
-            // return type). Skip adding a note in this case, since it
-            // would be incorrect.
-            if !err.span.primary_spans().iter().any(|&span| span == sp) {
-                let hir = fcx.tcx.hir();
-                let body_owner = hir.body_owned_by(hir.enclosing_body_owner(fcx.body_id));
-                if fcx.tcx.is_closure(hir.body_owner_def_id(body_owner).to_def_id()) {
-                    err.span_note(
-                        sp,
-                        &format!(
-                            "return type inferred to be `{}` here",
-                            fcx.resolve_vars_if_possible(expected)
-                        ),
-                    );
-                }
-            }
-        }
-
         err
     }
 
@@ -1554,7 +1539,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         if let hir::FnRetTy::Return(ty) = fn_output {
             // Get the return type.
             if let hir::TyKind::OpaqueDef(..) = ty.kind {
-                let ty = AstConv::ast_ty_to_ty(fcx, ty);
+                let ty = <dyn AstConv<'_>>::ast_ty_to_ty(fcx, ty);
                 // Get the `impl Trait`'s `DefId`.
                 if let ty::Opaque(def_id, _) = ty.kind() {
                     let hir_id = fcx.tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
@@ -1616,7 +1601,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
     fn is_return_ty_unsized(&self, fcx: &FnCtxt<'a, 'tcx>, blk_id: hir::HirId) -> bool {
         if let Some((fn_decl, _)) = fcx.get_fn_decl(blk_id) {
             if let hir::FnRetTy::Return(ty) = fn_decl.output {
-                let ty = AstConv::ast_ty_to_ty(fcx, ty);
+                let ty = <dyn AstConv<'_>>::ast_ty_to_ty(fcx, ty);
                 if let ty::Dynamic(..) = ty.kind() {
                     return true;
                 }
diff --git a/compiler/rustc_typeck/src/check/compare_method.rs b/compiler/rustc_typeck/src/check/compare_method.rs
index a30a8107933..60ca562f992 100644
--- a/compiler/rustc_typeck/src/check/compare_method.rs
+++ b/compiler/rustc_typeck/src/check/compare_method.rs
@@ -14,6 +14,7 @@ use rustc_middle::ty::{GenericParamDefKind, ToPredicate, TyCtxt};
 use rustc_span::Span;
 use rustc_trait_selection::traits::error_reporting::InferCtxtExt;
 use rustc_trait_selection::traits::{self, ObligationCause, ObligationCauseCode, Reveal};
+use std::iter;
 
 use super::{potentially_plural_count, FnCtxt, Inherited};
 
@@ -224,7 +225,7 @@ fn compare_predicate_entailment<'tcx>(
         let (impl_m_own_bounds, _) = infcx.replace_bound_vars_with_fresh_vars(
             impl_m_span,
             infer::HigherRankedType,
-            ty::Binder::bind(impl_m_own_bounds.predicates),
+            ty::Binder::bind(impl_m_own_bounds.predicates, tcx),
         );
         for predicate in impl_m_own_bounds {
             let traits::Normalized { value: predicate, obligations } =
@@ -257,14 +258,14 @@ fn compare_predicate_entailment<'tcx>(
         );
         let impl_sig =
             inh.normalize_associated_types_in(impl_m_span, impl_m_hir_id, param_env, impl_sig);
-        let impl_fty = tcx.mk_fn_ptr(ty::Binder::bind(impl_sig));
+        let impl_fty = tcx.mk_fn_ptr(ty::Binder::bind(impl_sig, tcx));
         debug!("compare_impl_method: impl_fty={:?}", impl_fty);
 
         let trait_sig = tcx.liberate_late_bound_regions(impl_m.def_id, tcx.fn_sig(trait_m.def_id));
         let trait_sig = trait_sig.subst(tcx, trait_to_placeholder_substs);
         let trait_sig =
             inh.normalize_associated_types_in(impl_m_span, impl_m_hir_id, param_env, trait_sig);
-        let trait_fty = tcx.mk_fn_ptr(ty::Binder::bind(trait_sig));
+        let trait_fty = tcx.mk_fn_ptr(ty::Binder::bind(trait_sig, tcx));
 
         debug!("compare_impl_method: trait_fty={:?}", trait_fty);
 
@@ -277,9 +278,8 @@ fn compare_predicate_entailment<'tcx>(
         if let Err(terr) = sub_result {
             debug!("sub_types failed: impl ty {:?}, trait ty {:?}", impl_fty, trait_fty);
 
-            let (impl_err_span, trait_err_span) = extract_spans_for_error_reporting(
-                &infcx, param_env, &terr, &cause, impl_m, impl_sig, trait_m, trait_sig,
-            );
+            let (impl_err_span, trait_err_span) =
+                extract_spans_for_error_reporting(&infcx, &terr, &cause, impl_m, trait_m);
 
             cause.make_mut().span = impl_err_span;
 
@@ -290,18 +290,79 @@ fn compare_predicate_entailment<'tcx>(
                 "method `{}` has an incompatible type for trait",
                 trait_m.ident
             );
-            if let TypeError::Mutability = terr {
-                if let Some(trait_err_span) = trait_err_span {
-                    if let Ok(trait_err_str) = tcx.sess.source_map().span_to_snippet(trait_err_span)
+            match &terr {
+                TypeError::ArgumentMutability(0) | TypeError::ArgumentSorts(_, 0)
+                    if trait_m.fn_has_self_parameter =>
+                {
+                    let ty = trait_sig.inputs()[0];
+                    let sugg = match ExplicitSelf::determine(ty, |_| ty == impl_trait_ref.self_ty())
                     {
+                        ExplicitSelf::ByValue => "self".to_owned(),
+                        ExplicitSelf::ByReference(_, hir::Mutability::Not) => "&self".to_owned(),
+                        ExplicitSelf::ByReference(_, hir::Mutability::Mut) => {
+                            "&mut self".to_owned()
+                        }
+                        _ => format!("self: {}", ty),
+                    };
+
+                    // When the `impl` receiver is an arbitrary self type, like `self: Box<Self>`, the
+                    // span points only at the type `Box<Self`>, but we want to cover the whole
+                    // argument pattern and type.
+                    let impl_m_hir_id =
+                        tcx.hir().local_def_id_to_hir_id(impl_m.def_id.expect_local());
+                    let span = match tcx.hir().expect_impl_item(impl_m_hir_id).kind {
+                        ImplItemKind::Fn(ref sig, body) => tcx
+                            .hir()
+                            .body_param_names(body)
+                            .zip(sig.decl.inputs.iter())
+                            .map(|(param, ty)| param.span.to(ty.span))
+                            .next()
+                            .unwrap_or(impl_err_span),
+                        _ => bug!("{:?} is not a method", impl_m),
+                    };
+
+                    diag.span_suggestion(
+                        span,
+                        "change the self-receiver type to match the trait",
+                        sugg,
+                        Applicability::MachineApplicable,
+                    );
+                }
+                TypeError::ArgumentMutability(i) | TypeError::ArgumentSorts(_, i) => {
+                    if trait_sig.inputs().len() == *i {
+                        // Suggestion to change output type. We do not suggest in `async` functions
+                        // to avoid complex logic or incorrect output.
+                        let impl_m_hir_id =
+                            tcx.hir().local_def_id_to_hir_id(impl_m.def_id.expect_local());
+                        match tcx.hir().expect_impl_item(impl_m_hir_id).kind {
+                            ImplItemKind::Fn(ref sig, _)
+                                if sig.header.asyncness == hir::IsAsync::NotAsync =>
+                            {
+                                let msg = "change the output type to match the trait";
+                                let ap = Applicability::MachineApplicable;
+                                match sig.decl.output {
+                                    hir::FnRetTy::DefaultReturn(sp) => {
+                                        let sugg = format!("-> {} ", trait_sig.output());
+                                        diag.span_suggestion_verbose(sp, msg, sugg, ap);
+                                    }
+                                    hir::FnRetTy::Return(hir_ty) => {
+                                        let sugg = trait_sig.output().to_string();
+                                        diag.span_suggestion(hir_ty.span, msg, sugg, ap);
+                                    }
+                                };
+                            }
+                            _ => {}
+                        };
+                    } else if let Some(trait_ty) = trait_sig.inputs().get(*i) {
                         diag.span_suggestion(
                             impl_err_span,
-                            "consider changing the mutability to match the trait",
-                            trait_err_str,
+                            "change the parameter type to match the trait",
+                            trait_ty.to_string(),
                             Applicability::MachineApplicable,
                         );
                     }
                 }
+                _ => {}
             }
 
             infcx.note_type_err(
@@ -384,90 +445,35 @@ fn check_region_bounds_on_impl_item<'tcx>(
 
 fn extract_spans_for_error_reporting<'a, 'tcx>(
     infcx: &infer::InferCtxt<'a, 'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
     terr: &TypeError<'_>,
     cause: &ObligationCause<'tcx>,
     impl_m: &ty::AssocItem,
-    impl_sig: ty::FnSig<'tcx>,
     trait_m: &ty::AssocItem,
-    trait_sig: ty::FnSig<'tcx>,
 ) -> (Span, Option<Span>) {
     let tcx = infcx.tcx;
     let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m.def_id.expect_local());
-    let (impl_m_output, impl_m_iter) = match tcx.hir().expect_impl_item(impl_m_hir_id).kind {
-        ImplItemKind::Fn(ref impl_m_sig, _) => {
-            (&impl_m_sig.decl.output, impl_m_sig.decl.inputs.iter())
+    let mut impl_args = match tcx.hir().expect_impl_item(impl_m_hir_id).kind {
+        ImplItemKind::Fn(ref sig, _) => {
+            sig.decl.inputs.iter().map(|t| t.span).chain(iter::once(sig.decl.output.span()))
         }
         _ => bug!("{:?} is not a method", impl_m),
     };
-
-    match *terr {
-        TypeError::Mutability => {
-            if let Some(def_id) = trait_m.def_id.as_local() {
-                let trait_m_hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-                let trait_m_iter = match tcx.hir().expect_trait_item(trait_m_hir_id).kind {
-                    TraitItemKind::Fn(ref trait_m_sig, _) => trait_m_sig.decl.inputs.iter(),
-                    _ => bug!("{:?} is not a TraitItemKind::Fn", trait_m),
-                };
-
-                impl_m_iter
-                    .zip(trait_m_iter)
-                    .find(|&(ref impl_arg, ref trait_arg)| {
-                        match (&impl_arg.kind, &trait_arg.kind) {
-                            (
-                                &hir::TyKind::Rptr(_, ref impl_mt),
-                                &hir::TyKind::Rptr(_, ref trait_mt),
-                            )
-                            | (&hir::TyKind::Ptr(ref impl_mt), &hir::TyKind::Ptr(ref trait_mt)) => {
-                                impl_mt.mutbl != trait_mt.mutbl
-                            }
-                            _ => false,
-                        }
-                    })
-                    .map(|(ref impl_arg, ref trait_arg)| (impl_arg.span, Some(trait_arg.span)))
-                    .unwrap_or_else(|| (cause.span(tcx), tcx.hir().span_if_local(trait_m.def_id)))
-            } else {
-                (cause.span(tcx), tcx.hir().span_if_local(trait_m.def_id))
+    let trait_args = trait_m.def_id.as_local().map(|def_id| {
+        let trait_m_hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        match tcx.hir().expect_trait_item(trait_m_hir_id).kind {
+            TraitItemKind::Fn(ref sig, _) => {
+                sig.decl.inputs.iter().map(|t| t.span).chain(iter::once(sig.decl.output.span()))
             }
+            _ => bug!("{:?} is not a TraitItemKind::Fn", trait_m),
         }
-        TypeError::Sorts(ExpectedFound { .. }) => {
-            if let Some(def_id) = trait_m.def_id.as_local() {
-                let trait_m_hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-                let (trait_m_output, trait_m_iter) =
-                    match tcx.hir().expect_trait_item(trait_m_hir_id).kind {
-                        TraitItemKind::Fn(ref trait_m_sig, _) => {
-                            (&trait_m_sig.decl.output, trait_m_sig.decl.inputs.iter())
-                        }
-                        _ => bug!("{:?} is not a TraitItemKind::Fn", trait_m),
-                    };
+    });
 
-                let impl_iter = impl_sig.inputs().iter();
-                let trait_iter = trait_sig.inputs().iter();
-                impl_iter
-                    .zip(trait_iter)
-                    .zip(impl_m_iter)
-                    .zip(trait_m_iter)
-                    .find_map(|(((&impl_arg_ty, &trait_arg_ty), impl_arg), trait_arg)| match infcx
-                        .at(&cause, param_env)
-                        .sub(trait_arg_ty, impl_arg_ty)
-                    {
-                        Ok(_) => None,
-                        Err(_) => Some((impl_arg.span, Some(trait_arg.span))),
-                    })
-                    .unwrap_or_else(|| {
-                        if infcx
-                            .at(&cause, param_env)
-                            .sup(trait_sig.output(), impl_sig.output())
-                            .is_err()
-                        {
-                            (impl_m_output.span(), Some(trait_m_output.span()))
-                        } else {
-                            (cause.span(tcx), tcx.hir().span_if_local(trait_m.def_id))
-                        }
-                    })
-            } else {
-                (cause.span(tcx), tcx.hir().span_if_local(trait_m.def_id))
-            }
+    match *terr {
+        TypeError::ArgumentMutability(i) => {
+            (impl_args.nth(i).unwrap(), trait_args.and_then(|mut args| args.nth(i)))
+        }
+        TypeError::ArgumentSorts(ExpectedFound { .. }, i) => {
+            (impl_args.nth(i).unwrap(), trait_args.and_then(|mut args| args.nth(i)))
         }
         _ => (cause.span(tcx), tcx.hir().span_if_local(trait_m.def_id)),
     }
@@ -517,8 +523,7 @@ fn compare_self_type<'tcx>(
                 tcx.sess,
                 impl_m_span,
                 E0185,
-                "method `{}` has a `{}` declaration in the impl, but \
-                                            not in the trait",
+                "method `{}` has a `{}` declaration in the impl, but not in the trait",
                 trait_m.ident,
                 self_descr
             );
@@ -538,8 +543,7 @@ fn compare_self_type<'tcx>(
                 tcx.sess,
                 impl_m_span,
                 E0186,
-                "method `{}` has a `{}` declaration in the trait, but \
-                                            not in the impl",
+                "method `{}` has a `{}` declaration in the trait, but not in the impl",
                 trait_m.ident,
                 self_descr
             );
@@ -792,14 +796,14 @@ fn compare_synthetic_generics<'tcx>(
     let trait_m_generics = tcx.generics_of(trait_m.def_id);
     let impl_m_type_params = impl_m_generics.params.iter().filter_map(|param| match param.kind {
         GenericParamDefKind::Type { synthetic, .. } => Some((param.def_id, synthetic)),
-        GenericParamDefKind::Lifetime | GenericParamDefKind::Const => None,
+        GenericParamDefKind::Lifetime | GenericParamDefKind::Const { .. } => None,
     });
     let trait_m_type_params = trait_m_generics.params.iter().filter_map(|param| match param.kind {
         GenericParamDefKind::Type { synthetic, .. } => Some((param.def_id, synthetic)),
-        GenericParamDefKind::Lifetime | GenericParamDefKind::Const => None,
+        GenericParamDefKind::Lifetime | GenericParamDefKind::Const { .. } => None,
     });
     for ((impl_def_id, impl_synthetic), (trait_def_id, trait_synthetic)) in
-        impl_m_type_params.zip(trait_m_type_params)
+        iter::zip(impl_m_type_params, trait_m_type_params)
     {
         if impl_synthetic != trait_synthetic {
             let impl_hir_id = tcx.hir().local_def_id_to_hir_id(impl_def_id.expect_local());
@@ -996,8 +1000,7 @@ crate fn compare_const_impl<'tcx>(
                 tcx.sess,
                 cause.span,
                 E0326,
-                "implemented const `{}` has an incompatible type for \
-                                             trait",
+                "implemented const `{}` has an incompatible type for trait",
                 trait_c.ident
             );
 
diff --git a/compiler/rustc_typeck/src/check/demand.rs b/compiler/rustc_typeck/src/check/demand.rs
index 39b973ed371..e5fcdcfa743 100644
--- a/compiler/rustc_typeck/src/check/demand.rs
+++ b/compiler/rustc_typeck/src/check/demand.rs
@@ -37,6 +37,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.suggest_missing_parentheses(err, expr);
         self.note_need_for_fn_pointer(err, expected, expr_ty);
         self.note_internal_mutation_in_method(err, expr, expected, expr_ty);
+        self.report_closure_infered_return_type(err, expected)
     }
 
     // Requires that the two types unify, and prints an error message if
@@ -200,7 +201,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     if self.can_coerce(expr_ty, sole_field_ty) {
                         let variant_path = self.tcx.def_path_str(variant.def_id);
                         // FIXME #56861: DRYer prelude filtering
-                        Some(variant_path.trim_start_matches("std::prelude::v1::").to_string())
+                        if let Some(path) = variant_path.strip_prefix("std::prelude::") {
+                            if let Some((_, path)) = path.split_once("::") {
+                                return Some(path.to_string());
+                            }
+                        }
+                        Some(variant_path)
                     } else {
                         None
                     }
@@ -361,6 +367,29 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         false
     }
 
+    /// If the given `HirId` corresponds to a block with a trailing expression, return that expression
+    crate fn maybe_get_block_expr(&self, hir_id: hir::HirId) -> Option<&'tcx hir::Expr<'tcx>> {
+        match self.tcx.hir().find(hir_id)? {
+            Node::Expr(hir::Expr { kind: hir::ExprKind::Block(block, ..), .. }) => block.expr,
+            _ => None,
+        }
+    }
+
+    /// Returns whether the given expression is an `else if`.
+    crate fn is_else_if_block(&self, expr: &hir::Expr<'_>) -> bool {
+        if let hir::ExprKind::If(..) = expr.kind {
+            let parent_id = self.tcx.hir().get_parent_node(expr.hir_id);
+            if let Some(Node::Expr(hir::Expr {
+                kind: hir::ExprKind::If(_, _, Some(else_expr)),
+                ..
+            })) = self.tcx.hir().find(parent_id)
+            {
+                return else_expr.hir_id == expr.hir_id;
+            }
+        }
+        false
+    }
+
     /// This function is used to determine potential "simple" improvements or users' errors and
     /// provide them useful help. For example:
     ///
@@ -647,6 +676,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 };
                                 let suggestion = if is_struct_pat_shorthand_field {
                                     format!("{}: *{}", code, code)
+                                } else if self.is_else_if_block(expr) {
+                                    // Don't suggest nonsense like `else *if`
+                                    return None;
+                                } else if let Some(expr) = self.maybe_get_block_expr(expr.hir_id) {
+                                    format!("*{}", sm.span_to_snippet(expr.span).unwrap_or(code))
                                 } else {
                                     format!("*{}", code)
                                 };
@@ -1028,4 +1062,32 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             _ => false,
         }
     }
+
+    // Report the type inferred by the return statement.
+    fn report_closure_infered_return_type(
+        &self,
+        err: &mut DiagnosticBuilder<'_>,
+        expected: Ty<'tcx>,
+    ) {
+        if let Some(sp) = self.ret_coercion_span.get() {
+            // If the closure has an explicit return type annotation,
+            // then a type error may occur at the first return expression we
+            // see in the closure (if it conflicts with the declared
+            // return type). Skip adding a note in this case, since it
+            // would be incorrect.
+            if !err.span.primary_spans().iter().any(|&span| span == sp) {
+                let hir = self.tcx.hir();
+                let body_owner = hir.body_owned_by(hir.enclosing_body_owner(self.body_id));
+                if self.tcx.is_closure(hir.body_owner_def_id(body_owner).to_def_id()) {
+                    err.span_note(
+                        sp,
+                        &format!(
+                            "return type inferred to be `{}` here",
+                            self.resolve_vars_if_possible(expected)
+                        ),
+                    );
+                }
+            }
+        }
+    }
 }
diff --git a/compiler/rustc_typeck/src/check/dropck.rs b/compiler/rustc_typeck/src/check/dropck.rs
index 4c3c4fd4470..de6336b254b 100644
--- a/compiler/rustc_typeck/src/check/dropck.rs
+++ b/compiler/rustc_typeck/src/check/dropck.rs
@@ -77,7 +77,7 @@ fn ensure_drop_params_and_item_params_correspond<'tcx>(
     tcx.infer_ctxt().enter(|ref infcx| {
         let impl_param_env = tcx.param_env(self_type_did);
         let tcx = infcx.tcx;
-        let mut fulfillment_cx = TraitEngine::new(tcx);
+        let mut fulfillment_cx = <dyn TraitEngine<'_>>::new(tcx);
 
         let named_type = tcx.type_of(self_type_did);
 
@@ -354,9 +354,9 @@ impl TypeRelation<'tcx> for SimpleEqRelation<'tcx> {
 
     fn binders<T>(
         &mut self,
-        a: ty::Binder<T>,
-        b: ty::Binder<T>,
-    ) -> RelateResult<'tcx, ty::Binder<T>>
+        a: ty::Binder<'tcx, T>,
+        b: ty::Binder<'tcx, T>,
+    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
     where
         T: Relate<'tcx>,
     {
diff --git a/compiler/rustc_typeck/src/check/expectation.rs b/compiler/rustc_typeck/src/check/expectation.rs
index 22be10a731f..e9e81034477 100644
--- a/compiler/rustc_typeck/src/check/expectation.rs
+++ b/compiler/rustc_typeck/src/check/expectation.rs
@@ -104,8 +104,8 @@ impl<'a, 'tcx> Expectation<'tcx> {
     /// for the program to type-check). `only_has_type` will return
     /// such a constraint, if it exists.
     pub(super) fn only_has_type(self, fcx: &FnCtxt<'a, 'tcx>) -> Option<Ty<'tcx>> {
-        match self.resolve(fcx) {
-            ExpectHasType(ty) => Some(ty),
+        match self {
+            ExpectHasType(ty) => Some(fcx.resolve_vars_if_possible(ty)),
             NoExpectation | ExpectCastableToType(_) | ExpectRvalueLikeUnsized(_) | IsLast(_) => {
                 None
             }
diff --git a/compiler/rustc_typeck/src/check/expr.rs b/compiler/rustc_typeck/src/check/expr.rs
index 2faf128c491..991c2ba693d 100644
--- a/compiler/rustc_typeck/src/check/expr.rs
+++ b/compiler/rustc_typeck/src/check/expr.rs
@@ -6,7 +6,7 @@ use crate::astconv::AstConv as _;
 use crate::check::cast;
 use crate::check::coercion::CoerceMany;
 use crate::check::fatally_break_rust;
-use crate::check::method::{probe, MethodError, SelfSource};
+use crate::check::method::SelfSource;
 use crate::check::report_unexpected_variant_res;
 use crate::check::BreakableCtxt;
 use crate::check::Diverges;
@@ -30,7 +30,6 @@ use rustc_errors::{pluralize, struct_span_err, Applicability, DiagnosticBuilder,
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, DefKind, Res};
 use rustc_hir::def_id::DefId;
-use rustc_hir::lang_items::LangItem;
 use rustc_hir::{ExprKind, QPath};
 use rustc_infer::infer;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
@@ -162,7 +161,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expr: &'tcx hir::Expr<'tcx>,
         expected: Expectation<'tcx>,
     ) -> Ty<'tcx> {
-        debug!(">> type-checking: expr={:?} expected={:?}", expr, expected);
+        debug!(">> type-checking: expected={:?}, expr={:?} ", expected, expr);
 
         // True if `expr` is a `Try::from_ok(())` that is a result of desugaring a try block
         // without the final expr (e.g. `try { return; }`). We don't want to generate an
@@ -225,7 +224,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expr: &'tcx hir::Expr<'tcx>,
         expected: Expectation<'tcx>,
     ) -> Ty<'tcx> {
-        debug!("check_expr_kind(expr={:?}, expected={:?})", expr, expected);
+        debug!("check_expr_kind(expected={:?}, expr={:?})", expected, expr);
 
         let tcx = self.tcx;
         match expr.kind {
@@ -461,7 +460,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.resolve_lang_item_path(lang_item, expr.span, expr.hir_id).1
     }
 
-    fn check_expr_path(&self, qpath: &hir::QPath<'_>, expr: &'tcx hir::Expr<'tcx>) -> Ty<'tcx> {
+    fn check_expr_path(
+        &self,
+        qpath: &'tcx hir::QPath<'tcx>,
+        expr: &'tcx hir::Expr<'tcx>,
+    ) -> Ty<'tcx> {
         let tcx = self.tcx;
         let (res, opt_ty, segs) = self.resolve_ty_and_res_ufcs(qpath, expr.hir_id, expr.span);
         let ty = match res {
@@ -600,7 +603,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         &cause,
                         &mut |mut err| {
                             self.suggest_mismatched_types_on_tail(
-                                &mut err, expr, ty, e_ty, cause.span, target_id,
+                                &mut err, expr, ty, e_ty, target_id,
                             );
                             if let Some(val) = ty_kind_suggestion(ty) {
                                 let label = destination
@@ -711,7 +714,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         });
 
         let ret_ty = ret_coercion.borrow().expected_ty();
-        let return_expr_ty = self.check_expr_with_hint(return_expr, ret_ty.clone());
+        let return_expr_ty = self.check_expr_with_hint(return_expr, ret_ty);
         ret_coercion.borrow_mut().coerce(
             self,
             &self.cause(return_expr.span, ObligationCauseCode::ReturnValue(return_expr.hir_id)),
@@ -947,7 +950,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
             Err(error) => {
                 if segment.ident.name != kw::Empty {
-                    self.report_extended_method_error(segment, span, args, rcvr_t, error);
+                    if let Some(mut err) = self.report_method_error(
+                        span,
+                        rcvr_t,
+                        segment.ident,
+                        SelfSource::MethodCall(&args[0]),
+                        error,
+                        Some(args),
+                    ) {
+                        err.emit();
+                    }
                 }
                 Err(())
             }
@@ -964,59 +976,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         )
     }
 
-    fn report_extended_method_error(
-        &self,
-        segment: &hir::PathSegment<'_>,
-        span: Span,
-        args: &'tcx [hir::Expr<'tcx>],
-        rcvr_t: Ty<'tcx>,
-        error: MethodError<'tcx>,
-    ) {
-        let rcvr = &args[0];
-        let try_alt_rcvr = |err: &mut DiagnosticBuilder<'_>, new_rcvr_t| {
-            if let Some(new_rcvr_t) = new_rcvr_t {
-                if let Ok(pick) = self.lookup_probe(
-                    span,
-                    segment.ident,
-                    new_rcvr_t,
-                    rcvr,
-                    probe::ProbeScope::AllTraits,
-                ) {
-                    debug!("try_alt_rcvr: pick candidate {:?}", pick);
-                    // Make sure the method is defined for the *actual* receiver:
-                    // we don't want to treat `Box<Self>` as a receiver if
-                    // it only works because of an autoderef to `&self`
-                    if pick.autoderefs == 0 {
-                        err.span_label(
-                            pick.item.ident.span,
-                            &format!("the method is available for `{}` here", new_rcvr_t),
-                        );
-                    }
-                }
-            }
-        };
-
-        if let Some(mut err) = self.report_method_error(
-            span,
-            rcvr_t,
-            segment.ident,
-            SelfSource::MethodCall(rcvr),
-            error,
-            Some(args),
-        ) {
-            if let ty::Adt(..) = rcvr_t.kind() {
-                // Try alternative arbitrary self types that could fulfill this call.
-                // FIXME: probe for all types that *could* be arbitrary self-types, not
-                // just this list.
-                try_alt_rcvr(&mut err, self.tcx.mk_lang_item(rcvr_t, LangItem::OwnedBox));
-                try_alt_rcvr(&mut err, self.tcx.mk_lang_item(rcvr_t, LangItem::Pin));
-                try_alt_rcvr(&mut err, self.tcx.mk_diagnostic_item(rcvr_t, sym::Arc));
-                try_alt_rcvr(&mut err, self.tcx.mk_diagnostic_item(rcvr_t, sym::Rc));
-            }
-            err.emit();
-        }
-    }
-
     fn check_expr_cast(
         &self,
         e: &'tcx hir::Expr<'tcx>,
@@ -1158,7 +1117,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expr: &hir::Expr<'_>,
         expected: Expectation<'tcx>,
         qpath: &QPath<'_>,
-        fields: &'tcx [hir::Field<'tcx>],
+        fields: &'tcx [hir::ExprField<'tcx>],
         base_expr: &'tcx Option<&'tcx hir::Expr<'tcx>>,
     ) -> Ty<'tcx> {
         // Find the relevant variant
@@ -1231,7 +1190,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expr_id: hir::HirId,
         span: Span,
         variant: &'tcx ty::VariantDef,
-        ast_fields: &'tcx [hir::Field<'tcx>],
+        ast_fields: &'tcx [hir::ExprField<'tcx>],
         check_completeness: bool,
     ) -> bool {
         let tcx = self.tcx;
@@ -1320,7 +1279,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     fn check_struct_fields_on_error(
         &self,
-        fields: &'tcx [hir::Field<'tcx>],
+        fields: &'tcx [hir::ExprField<'tcx>],
         base_expr: &'tcx Option<&'tcx hir::Expr<'tcx>>,
     ) {
         for field in fields {
@@ -1411,8 +1370,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         ty: Ty<'tcx>,
         variant: &'tcx ty::VariantDef,
-        field: &hir::Field<'_>,
-        skip_fields: &[hir::Field<'_>],
+        field: &hir::ExprField<'_>,
+        skip_fields: &[hir::ExprField<'_>],
         kind_name: &str,
         ty_span: Span,
     ) {
@@ -2128,7 +2087,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn check_expr_asm(&self, asm: &'tcx hir::InlineAsm<'tcx>) -> Ty<'tcx> {
         for (op, _op_sp) in asm.operands {
             match op {
-                hir::InlineAsmOperand::In { expr, .. } | hir::InlineAsmOperand::Const { expr } => {
+                hir::InlineAsmOperand::In { expr, .. } => {
                     self.check_expr_asm_operand(expr, true);
                 }
                 hir::InlineAsmOperand::Out { expr, .. } => {
@@ -2145,6 +2104,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         self.check_expr_asm_operand(out_expr, false);
                     }
                 }
+                hir::InlineAsmOperand::Const { anon_const } => {
+                    self.to_const(anon_const);
+                }
                 hir::InlineAsmOperand::Sym { expr } => {
                     self.check_expr(expr);
                 }
diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs
index f5e9cc1efcc..9ace4550421 100644
--- a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs
+++ b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs
@@ -6,6 +6,7 @@ use crate::check::callee::{self, DeferredCallResolution};
 use crate::check::method::{self, MethodCallee, SelfSource};
 use crate::check::{BreakableCtxt, Diverges, Expectation, FallbackMode, FnCtxt, LocalTy};
 
+use rustc_ast::TraitObjectSyntax;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{Applicability, DiagnosticBuilder, ErrorReported};
@@ -13,7 +14,7 @@ use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItem;
-use rustc_hir::{ExprKind, GenericArg, Node, QPath};
+use rustc_hir::{ExprKind, GenericArg, Node, QPath, TyKind};
 use rustc_infer::infer::canonical::{Canonical, OriginalQueryValues, QueryResponse};
 use rustc_infer::infer::error_reporting::TypeAnnotationNeeded::E0282;
 use rustc_infer::infer::{InferOk, InferResult};
@@ -27,10 +28,12 @@ use rustc_middle::ty::{
     Ty, UserType,
 };
 use rustc_session::lint;
-use rustc_span::hygiene::DesugaringKind;
+use rustc_session::lint::builtin::BARE_TRAIT_OBJECTS;
+use rustc_session::parse::feature_err;
 use rustc_span::source_map::{original_sp, DUMMY_SP};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::{self, BytePos, MultiSpan, Span};
+use rustc_span::{hygiene::DesugaringKind, Symbol};
 use rustc_trait_selection::infer::InferCtxtExt as _;
 use rustc_trait_selection::opaque_types::InferCtxtExt as _;
 use rustc_trait_selection::traits::error_reporting::InferCtxtExt as _;
@@ -39,6 +42,7 @@ use rustc_trait_selection::traits::{
 };
 
 use std::collections::hash_map::Entry;
+use std::iter;
 use std::slice;
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
@@ -362,6 +366,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         parent_id: hir::HirId,
         value: T,
         value_span: Span,
+        feature: Option<Symbol>,
     ) -> T {
         let parent_def_id = self.tcx.hir().local_def_id(parent_id);
         debug!(
@@ -380,7 +385,21 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let mut opaque_types = self.opaque_types.borrow_mut();
         let mut opaque_types_vars = self.opaque_types_vars.borrow_mut();
+
         for (ty, decl) in opaque_type_map {
+            if let Some(feature) = feature {
+                if let hir::OpaqueTyOrigin::TyAlias = decl.origin {
+                    if !self.tcx.features().enabled(feature) {
+                        feature_err(
+                            &self.tcx.sess.parse_sess,
+                            feature,
+                            value_span,
+                            "type alias impl trait is not permitted here",
+                        )
+                        .emit();
+                    }
+                }
+            }
             let _ = opaque_types.insert(ty, decl);
             let _ = opaque_types_vars.insert(decl.concrete_ty, decl.opaque_type);
         }
@@ -457,7 +476,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     pub fn to_ty(&self, ast_t: &hir::Ty<'_>) -> Ty<'tcx> {
-        let t = AstConv::ast_ty_to_ty(self, ast_t);
+        let t = <dyn AstConv<'_>>::ast_ty_to_ty(self, ast_t);
         self.register_wf_obligation(t.into(), ast_t.span, traits::MiscObligation);
         t
     }
@@ -839,7 +858,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         // out unconstrained or ambiguous, as we're
                         // just trying to get hints here.
                         self.save_and_restore_in_snapshot_flag(|_| {
-                            let mut fulfill = TraitEngine::new(self.tcx);
+                            let mut fulfill = <dyn TraitEngine<'_>>::new(self.tcx);
                             for obligation in ok.obligations {
                                 fulfill.register_predicate_obligation(self, obligation);
                             }
@@ -886,12 +905,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     /// Resolves an associated value path into a base type and associated constant, or method
     /// resolution. The newly resolved definition is written into `type_dependent_defs`.
-    pub fn resolve_ty_and_res_ufcs<'b>(
+    pub fn resolve_ty_and_res_ufcs(
         &self,
-        qpath: &'b QPath<'b>,
+        qpath: &'tcx QPath<'tcx>,
         hir_id: hir::HirId,
         span: Span,
-    ) -> (Res, Option<Ty<'tcx>>, &'b [hir::PathSegment<'b>]) {
+    ) -> (Res, Option<Ty<'tcx>>, &'tcx [hir::PathSegment<'tcx>]) {
         debug!("resolve_ty_and_res_ufcs: qpath={:?} hir_id={:?} span={:?}", qpath, hir_id, span);
         let (ty, qself, item_segment) = match *qpath {
             QPath::Resolved(ref opt_qself, ref path) => {
@@ -932,6 +951,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             result
         });
 
+        if result.is_ok() {
+            self.maybe_lint_bare_trait(qpath, hir_id);
+        }
+
         // Write back the new resolution.
         self.write_resolution(hir_id, result);
         (
@@ -941,6 +964,29 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         )
     }
 
+    fn maybe_lint_bare_trait(&self, qpath: &QPath<'_>, hir_id: hir::HirId) {
+        if let QPath::TypeRelative(self_ty, _) = qpath {
+            if let TyKind::TraitObject([poly_trait_ref, ..], _, TraitObjectSyntax::None) =
+                self_ty.kind
+            {
+                self.tcx.struct_span_lint_hir(BARE_TRAIT_OBJECTS, hir_id, self_ty.span, |lint| {
+                    let mut db = lint
+                        .build(&format!("trait objects without an explicit `dyn` are deprecated"));
+                    let (sugg, app) = match self.tcx.sess.source_map().span_to_snippet(self_ty.span)
+                    {
+                        Ok(s) if poly_trait_ref.trait_ref.path.is_global() => {
+                            (format!("<dyn ({})>", s), Applicability::MachineApplicable)
+                        }
+                        Ok(s) => (format!("<dyn {}>", s), Applicability::MachineApplicable),
+                        Err(_) => ("<dyn <type>>".to_string(), Applicability::HasPlaceholders),
+                    };
+                    db.span_suggestion(self_ty.span, "use `dyn`", sugg, app);
+                    db.emit()
+                });
+            }
+        }
+    }
+
     /// Given a function `Node`, return its `FnDecl` if it exists, or `None` otherwise.
     pub(in super::super) fn get_node_fn_decl(
         &self,
@@ -1101,7 +1147,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     (
                         hir::ItemKind::OpaqueTy(hir::OpaqueTy { bounds: last_bounds, .. }),
                         hir::ItemKind::OpaqueTy(hir::OpaqueTy { bounds: exp_bounds, .. }),
-                    ) if last_bounds.iter().zip(exp_bounds.iter()).all(|(left, right)| {
+                    ) if iter::zip(*last_bounds, *exp_bounds).all(|(left, right)| {
                         match (left, right) {
                             (
                                 hir::GenericBound::Trait(tl, ml),
@@ -1159,9 +1205,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let path_segs = match res {
             Res::Local(_) | Res::SelfCtor(_) => vec![],
-            Res::Def(kind, def_id) => {
-                AstConv::def_ids_for_value_path_segments(self, segments, self_ty, kind, def_id)
-            }
+            Res::Def(kind, def_id) => <dyn AstConv<'_>>::def_ids_for_value_path_segments(
+                self, segments, self_ty, kind, def_id,
+            ),
             _ => bug!("instantiate_value_path on {:?}", res),
         };
 
@@ -1204,7 +1250,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // errors if type parameters are provided in an inappropriate place.
 
         let generic_segs: FxHashSet<_> = path_segs.iter().map(|PathSeg(_, index)| index).collect();
-        let generics_has_err = AstConv::prohibit_generics(
+        let generics_has_err = <dyn AstConv<'_>>::prohibit_generics(
             self,
             segments.iter().enumerate().filter_map(|(index, seg)| {
                 if !generic_segs.contains(&index) || is_alias_variant_ctor {
@@ -1247,7 +1293,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if let GenericArgCountResult {
                 correct: Err(GenericArgCountMismatch { reported: Some(_), .. }),
                 ..
-            } = AstConv::check_generic_arg_count_for_call(
+            } = <dyn AstConv<'_>>::check_generic_arg_count_for_call(
                 tcx,
                 span,
                 def_id,
@@ -1355,12 +1401,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ) -> subst::GenericArg<'tcx> {
                 match (&param.kind, arg) {
                     (GenericParamDefKind::Lifetime, GenericArg::Lifetime(lt)) => {
-                        AstConv::ast_region_to_region(self.fcx, lt, Some(param)).into()
+                        <dyn AstConv<'_>>::ast_region_to_region(self.fcx, lt, Some(param)).into()
                     }
                     (GenericParamDefKind::Type { .. }, GenericArg::Type(ty)) => {
                         self.fcx.to_ty(ty).into()
                     }
-                    (GenericParamDefKind::Const, GenericArg::Const(ct)) => {
+                    (GenericParamDefKind::Const { .. }, GenericArg::Const(ct)) => {
                         self.fcx.const_arg_to_const(&ct.value, param.def_id).into()
                     }
                     _ => unreachable!(),
@@ -1398,23 +1444,25 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             self.fcx.var_for_def(self.span, param)
                         }
                     }
-                    GenericParamDefKind::Const => {
-                        // FIXME(const_generics_defaults)
-                        // No const parameters were provided, we have to infer them.
-                        self.fcx.var_for_def(self.span, param)
+                    GenericParamDefKind::Const { has_default, .. } => {
+                        if !infer_args && has_default {
+                            tcx.const_param_default(param.def_id).into()
+                        } else {
+                            self.fcx.var_for_def(self.span, param)
+                        }
                     }
                 }
             }
         }
 
         let substs = self_ctor_substs.unwrap_or_else(|| {
-            AstConv::create_substs_for_generic_args(
+            <dyn AstConv<'_>>::create_substs_for_generic_args(
                 tcx,
                 def_id,
                 &[][..],
                 has_self,
                 self_ty,
-                arg_count,
+                &arg_count,
                 &mut CreateCtorSubstsContext {
                     fcx: self,
                     span,
diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs
index 5b8b25c2100..80b5a9d4e62 100644
--- a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs
+++ b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs
@@ -23,6 +23,7 @@ use rustc_span::{self, MultiSpan, Span};
 use rustc_trait_selection::traits::{self, ObligationCauseCode, StatementAsExpression};
 
 use crate::structured_errors::StructuredDiagnostic;
+use std::iter;
 use std::slice;
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
@@ -108,7 +109,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // All the input types from the fn signature must outlive the call
         // so as to validate implied bounds.
-        for (&fn_input_ty, arg_expr) in fn_inputs.iter().zip(args.iter()) {
+        for (&fn_input_ty, arg_expr) in iter::zip(fn_inputs, args) {
             self.register_wf_obligation(fn_input_ty.into(), arg_expr.span, traits::MiscObligation);
         }
 
@@ -439,7 +440,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         qpath: &QPath<'_>,
         hir_id: hir::HirId,
     ) -> Option<(&'tcx ty::VariantDef, Ty<'tcx>)> {
-        let path_span = qpath.qself_span();
+        let path_span = qpath.span();
         let (def, ty) = self.finish_resolving_struct_path(qpath, path_span, hir_id);
         let variant = match def {
             Res::Err => {
@@ -875,7 +876,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         match *qpath {
             QPath::Resolved(ref maybe_qself, ref path) => {
                 let self_ty = maybe_qself.as_ref().map(|qself| self.to_ty(qself));
-                let ty = AstConv::res_to_ty(self, self_ty, path, true);
+                let ty = <dyn AstConv<'_>>::res_to_ty(self, self_ty, path, true);
                 (path.res, ty)
             }
             QPath::TypeRelative(ref qself, ref segment) => {
@@ -886,8 +887,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 } else {
                     Res::Err
                 };
-                let result =
-                    AstConv::associated_path_to_ty(self, hir_id, path_span, ty, res, segment, true);
+                let result = <dyn AstConv<'_>>::associated_path_to_ty(
+                    self, hir_id, path_span, ty, res, segment, true,
+                );
                 let ty = result.map(|(ty, _, _)| ty).unwrap_or_else(|_| self.tcx().ty_error());
                 let result = result.map(|(_, kind, def_id)| (kind, def_id));
 
@@ -1000,7 +1002,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                         // would trigger in `is_send::<T::AssocType>();`
                                         // from `typeck-default-trait-impl-assoc-type.rs`.
                                     } else {
-                                        let ty = AstConv::ast_ty_to_ty(self, hir_ty);
+                                        let ty = <dyn AstConv<'_>>::ast_ty_to_ty(self, hir_ty);
                                         let ty = self.resolve_vars_if_possible(ty);
                                         if ty == predicate.self_ty() {
                                             error.obligation.cause.make_mut().span = hir_ty.span;
diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs b/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs
index 9f15993e471..b7583344845 100644
--- a/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs
@@ -41,21 +41,21 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expr: &'tcx hir::Expr<'tcx>,
         expected: Ty<'tcx>,
         found: Ty<'tcx>,
-        cause_span: Span,
         blk_id: hir::HirId,
     ) -> bool {
         let expr = expr.peel_drop_temps();
         // If the expression is from an external macro, then do not suggest
         // adding a semicolon, because there's nowhere to put it.
         // See issue #81943.
-        if expr.can_have_side_effects() && !in_external_macro(self.tcx.sess, cause_span) {
-            self.suggest_missing_semicolon(err, expr, expected, cause_span);
+        if expr.can_have_side_effects() && !in_external_macro(self.tcx.sess, expr.span) {
+            self.suggest_missing_semicolon(err, expr, expected);
         }
         let mut pointing_at_return_type = false;
         if let Some((fn_decl, can_suggest)) = self.get_fn_decl(blk_id) {
             pointing_at_return_type =
                 self.suggest_missing_return_type(err, &fn_decl, expected, found, can_suggest);
-            self.suggest_missing_return_expr(err, expr, &fn_decl, expected, found);
+            let fn_id = self.tcx.hir().get_return_block(blk_id).unwrap();
+            self.suggest_missing_return_expr(err, expr, &fn_decl, expected, found, fn_id);
         }
         pointing_at_return_type
     }
@@ -204,6 +204,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         found: Ty<'tcx>,
         expected_ty_expr: Option<&'tcx hir::Expr<'tcx>>,
     ) {
+        let expr = expr.peel_blocks();
         if let Some((sp, msg, suggestion, applicability)) = self.check_ref(expr, found, expected) {
             err.span_suggestion(sp, msg, suggestion, applicability);
         } else if let (ty::FnDef(def_id, ..), true) =
@@ -218,8 +219,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.is_hir_id_from_struct_pattern_shorthand_field(expr.hir_id, expr.span);
             let methods = self.get_conversion_methods(expr.span, expected, found, expr.hir_id);
             if let Ok(expr_text) = self.sess().source_map().span_to_snippet(expr.span) {
-                let mut suggestions = iter::repeat(&expr_text)
-                    .zip(methods.iter())
+                let mut suggestions = iter::zip(iter::repeat(&expr_text), &methods)
                     .filter_map(|(receiver, method)| {
                         let method_call = format!(".{}()", method.ident);
                         if receiver.ends_with(&method_call) {
@@ -388,7 +388,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         err: &mut DiagnosticBuilder<'_>,
         expression: &'tcx hir::Expr<'tcx>,
         expected: Ty<'tcx>,
-        cause_span: Span,
     ) {
         if expected.is_unit() {
             // `BlockTailExpression` only relevant if the tail expr would be
@@ -403,7 +402,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     if expression.can_have_side_effects() =>
                 {
                     err.span_suggestion(
-                        cause_span.shrink_to_hi(),
+                        expression.span.shrink_to_hi(),
                         "consider using a semicolon here",
                         ";".to_string(),
                         Applicability::MachineApplicable,
@@ -461,7 +460,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // are not, the expectation must have been caused by something else.
                 debug!("suggest_missing_return_type: return type {:?} node {:?}", ty, ty.kind);
                 let sp = ty.span;
-                let ty = AstConv::ast_ty_to_ty(self, ty);
+                let ty = <dyn AstConv<'_>>::ast_ty_to_ty(self, ty);
                 debug!("suggest_missing_return_type: return type {:?}", ty);
                 debug!("suggest_missing_return_type: expected type {:?}", ty);
                 if ty.kind() == expected.kind() {
@@ -480,14 +479,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         fn_decl: &hir::FnDecl<'_>,
         expected: Ty<'tcx>,
         found: Ty<'tcx>,
+        id: hir::HirId,
     ) {
         if !expected.is_unit() {
             return;
         }
         let found = self.resolve_vars_with_obligations(found);
         if let hir::FnRetTy::Return(ty) = fn_decl.output {
-            let ty = AstConv::ast_ty_to_ty(self, ty);
-            let ty = self.tcx.erase_late_bound_regions(Binder::bind(ty));
+            let ty = <dyn AstConv<'_>>::ast_ty_to_ty(self, ty);
+            let bound_vars = self.tcx.late_bound_vars(id);
+            let ty = self.tcx.erase_late_bound_regions(Binder::bind_with_vars(ty, bound_vars));
             let ty = self.normalize_associated_types_in(expr.span, ty);
             if self.can_coerce(found, ty) {
                 err.multipart_suggestion(
diff --git a/compiler/rustc_typeck/src/check/gather_locals.rs b/compiler/rustc_typeck/src/check/gather_locals.rs
index 825ebc19fa6..2683e886eeb 100644
--- a/compiler/rustc_typeck/src/check/gather_locals.rs
+++ b/compiler/rustc_typeck/src/check/gather_locals.rs
@@ -4,9 +4,8 @@ use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_hir::PatKind;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_middle::ty::Ty;
-use rustc_span::Span;
+use rustc_span::{sym, Span};
 use rustc_trait_selection::traits;
-use std::mem;
 
 pub(super) struct GatherLocalsVisitor<'a, 'tcx> {
     fcx: &'a FnCtxt<'a, 'tcx>,
@@ -14,12 +13,12 @@ pub(super) struct GatherLocalsVisitor<'a, 'tcx> {
     // parameters are special cases of patterns, but we want to handle them as
     // *distinct* cases. so track when we are hitting a pattern *within* an fn
     // parameter.
-    outermost_fn_param_pat: bool,
+    outermost_fn_param_pat: Option<Span>,
 }
 
 impl<'a, 'tcx> GatherLocalsVisitor<'a, 'tcx> {
     pub(super) fn new(fcx: &'a FnCtxt<'a, 'tcx>, parent_id: hir::HirId) -> Self {
-        Self { fcx, parent_id, outermost_fn_param_pat: false }
+        Self { fcx, parent_id, outermost_fn_param_pat: None }
     }
 
     fn assign(&mut self, span: Span, nid: hir::HirId, ty_opt: Option<LocalTy<'tcx>>) -> Ty<'tcx> {
@@ -58,11 +57,12 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> {
             Some(ref ty) => {
                 let o_ty = self.fcx.to_ty(&ty);
 
-                let revealed_ty = if self.fcx.tcx.features().impl_trait_in_bindings {
-                    self.fcx.instantiate_opaque_types_from_value(self.parent_id, o_ty, ty.span)
-                } else {
-                    o_ty
-                };
+                let revealed_ty = self.fcx.instantiate_opaque_types_from_value(
+                    self.parent_id,
+                    o_ty,
+                    ty.span,
+                    Some(sym::impl_trait_in_bindings),
+                );
 
                 let c_ty =
                     self.fcx.inh.infcx.canonicalize_user_type_annotation(UserType::Ty(revealed_ty));
@@ -91,7 +91,7 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> {
     }
 
     fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) {
-        let old_outermost_fn_param_pat = mem::replace(&mut self.outermost_fn_param_pat, true);
+        let old_outermost_fn_param_pat = self.outermost_fn_param_pat.replace(param.ty_span);
         intravisit::walk_param(self, param);
         self.outermost_fn_param_pat = old_outermost_fn_param_pat;
     }
@@ -101,12 +101,12 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> {
         if let PatKind::Binding(_, _, ident, _) = p.kind {
             let var_ty = self.assign(p.span, p.hir_id, None);
 
-            if self.outermost_fn_param_pat {
+            if let Some(ty_span) = self.outermost_fn_param_pat {
                 if !self.fcx.tcx.features().unsized_fn_params {
                     self.fcx.require_type_is_sized(
                         var_ty,
                         p.span,
-                        traits::SizedArgumentType(Some(p.span)),
+                        traits::SizedArgumentType(Some(ty_span)),
                     );
                 }
             } else {
@@ -122,7 +122,7 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> {
                 var_ty
             );
         }
-        let old_outermost_fn_param_pat = mem::replace(&mut self.outermost_fn_param_pat, false);
+        let old_outermost_fn_param_pat = self.outermost_fn_param_pat.take();
         intravisit::walk_pat(self, p);
         self.outermost_fn_param_pat = old_outermost_fn_param_pat;
     }
diff --git a/compiler/rustc_typeck/src/check/generator_interior.rs b/compiler/rustc_typeck/src/check/generator_interior.rs
index 91708465b3f..e40aa914858 100644
--- a/compiler/rustc_typeck/src/check/generator_interior.rs
+++ b/compiler/rustc_typeck/src/check/generator_interior.rs
@@ -186,7 +186,10 @@ pub fn resolve_interior<'a, 'tcx>(
                 // which means that none of the regions inside relate to any other, even if
                 // typeck had previously found constraints that would cause them to be related.
                 let folded = fcx.tcx.fold_regions(erased, &mut false, |_, current_depth| {
-                    let br = ty::BoundRegion { kind: ty::BrAnon(counter) };
+                    let br = ty::BoundRegion {
+                        var: ty::BoundVar::from_u32(counter),
+                        kind: ty::BrAnon(counter),
+                    };
                     let r = fcx.tcx.mk_region(ty::ReLateBound(current_depth, br));
                     counter += 1;
                     r
@@ -202,11 +205,15 @@ pub fn resolve_interior<'a, 'tcx>(
 
     // Extract type components to build the witness type.
     let type_list = fcx.tcx.mk_type_list(type_causes.iter().map(|cause| cause.ty));
-    let witness = fcx.tcx.mk_generator_witness(ty::Binder::bind(type_list));
+    let bound_vars = fcx.tcx.mk_bound_variable_kinds(
+        (0..counter).map(|i| ty::BoundVariableKind::Region(ty::BrAnon(i))),
+    );
+    let witness =
+        fcx.tcx.mk_generator_witness(ty::Binder::bind_with_vars(type_list, bound_vars.clone()));
 
     // Store the generator types and spans into the typeck results for this generator.
     visitor.fcx.inh.typeck_results.borrow_mut().generator_interior_types =
-        ty::Binder::bind(type_causes);
+        ty::Binder::bind_with_vars(type_causes, bound_vars);
 
     debug!(
         "types in generator after region replacement {:?}, span = {:?}",
diff --git a/compiler/rustc_typeck/src/check/inherited.rs b/compiler/rustc_typeck/src/check/inherited.rs
index 0011a3fc71b..1dacbade1bd 100644
--- a/compiler/rustc_typeck/src/check/inherited.rs
+++ b/compiler/rustc_typeck/src/check/inherited.rs
@@ -117,7 +117,7 @@ impl Inherited<'a, 'tcx> {
                 maybe_typeck_results: infcx.in_progress_typeck_results,
             },
             infcx,
-            fulfillment_cx: RefCell::new(TraitEngine::new(tcx)),
+            fulfillment_cx: RefCell::new(<dyn TraitEngine<'_>>::new(tcx)),
             locals: RefCell::new(Default::default()),
             deferred_sized_obligations: RefCell::new(Vec::new()),
             deferred_call_resolutions: RefCell::new(Default::default()),
diff --git a/compiler/rustc_typeck/src/check/intrinsic.rs b/compiler/rustc_typeck/src/check/intrinsic.rs
index dedf96863ea..5741b6824b5 100644
--- a/compiler/rustc_typeck/src/check/intrinsic.rs
+++ b/compiler/rustc_typeck/src/check/intrinsic.rs
@@ -101,12 +101,21 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
     let intrinsic_name = tcx.item_name(it.def_id.to_def_id());
     let name_str = intrinsic_name.as_str();
 
+    let bound_vars = tcx.mk_bound_variable_kinds(
+        [ty::BoundVariableKind::Region(ty::BrAnon(0)), ty::BoundVariableKind::Region(ty::BrEnv)]
+            .iter()
+            .copied(),
+    );
     let mk_va_list_ty = |mutbl| {
         tcx.lang_items().va_list().map(|did| {
-            let region = tcx
-                .mk_region(ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { kind: ty::BrAnon(0) }));
-            let env_region =
-                tcx.mk_region(ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { kind: ty::BrEnv }));
+            let region = tcx.mk_region(ty::ReLateBound(
+                ty::INNERMOST,
+                ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0) },
+            ));
+            let env_region = tcx.mk_region(ty::ReLateBound(
+                ty::INNERMOST,
+                ty::BoundRegion { var: ty::BoundVar::from_u32(1), kind: ty::BrEnv },
+            ));
             let va_list_ty = tcx.type_of(did).subst(tcx, &[region.into()]);
             (tcx.mk_ref(env_region, ty::TypeAndMut { ty: va_list_ty, mutbl }), va_list_ty)
         })
@@ -305,7 +314,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                     tcx.associated_items(tcx.lang_items().discriminant_kind_trait().unwrap());
                 let discriminant_def_id = assoc_items.in_definition_order().next().unwrap().def_id;
 
-                let br = ty::BoundRegion { kind: ty::BrAnon(0) };
+                let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0) };
                 (
                     1,
                     vec![
@@ -366,7 +375,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
         (n_tps, inputs, output, unsafety)
     };
     let sig = tcx.mk_fn_sig(inputs.into_iter(), output, false, unsafety, Abi::RustIntrinsic);
-    let sig = ty::Binder::bind(sig);
+    let sig = ty::Binder::bind_with_vars(sig, bound_vars);
     equate_intrinsic_type(tcx, it, n_tps, sig)
 }
 
@@ -398,7 +407,8 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         | sym::simd_fpow
         | sym::simd_saturating_add
         | sym::simd_saturating_sub => (1, vec![param(0), param(0)], param(0)),
-        sym::simd_fsqrt
+        sym::simd_neg
+        | sym::simd_fsqrt
         | sym::simd_fsin
         | sym::simd_fcos
         | sym::simd_fexp
@@ -407,8 +417,10 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         | sym::simd_flog10
         | sym::simd_flog
         | sym::simd_fabs
+        | sym::simd_ceil
         | sym::simd_floor
-        | sym::simd_ceil => (1, vec![param(0)], param(0)),
+        | sym::simd_round
+        | sym::simd_trunc => (1, vec![param(0)], param(0)),
         sym::simd_fpowi => (1, vec![param(0), tcx.types.i32], param(0)),
         sym::simd_fma => (1, vec![param(0), param(0), param(0)], param(0)),
         sym::simd_gather => (3, vec![param(0), param(1), param(2)], param(0)),
diff --git a/compiler/rustc_typeck/src/check/method/confirm.rs b/compiler/rustc_typeck/src/check/method/confirm.rs
index e5f19281b07..f546a0d8963 100644
--- a/compiler/rustc_typeck/src/check/method/confirm.rs
+++ b/compiler/rustc_typeck/src/check/method/confirm.rs
@@ -15,6 +15,7 @@ use rustc_middle::ty::{self, GenericParamDefKind, Ty};
 use rustc_span::Span;
 use rustc_trait_selection::traits;
 
+use std::iter;
 use std::ops::Deref;
 
 struct ConfirmContext<'a, 'tcx> {
@@ -118,7 +119,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
         // We won't add these if we encountered an illegal sized bound, so that we can use
         // a custom error in that case.
         if illegal_sized_bound.is_none() {
-            let method_ty = self.tcx.mk_fn_ptr(ty::Binder::bind(method_sig));
+            let method_ty = self.tcx.mk_fn_ptr(ty::Binder::bind(method_sig, self.tcx));
             self.add_obligations(method_ty, all_substs, method_predicates);
         }
 
@@ -155,32 +156,46 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
         let mut target =
             self.structurally_resolved_type(autoderef.span(), autoderef.final_ty(false));
 
-        if let Some(mutbl) = pick.autoref {
-            let region = self.next_region_var(infer::Autoref(self.span, pick.item));
-            target = self.tcx.mk_ref(region, ty::TypeAndMut { mutbl, ty: target });
-            let mutbl = match mutbl {
-                hir::Mutability::Not => AutoBorrowMutability::Not,
-                hir::Mutability::Mut => AutoBorrowMutability::Mut {
-                    // Method call receivers are the primary use case
-                    // for two-phase borrows.
-                    allow_two_phase_borrow: AllowTwoPhase::Yes,
-                },
-            };
-            adjustments
-                .push(Adjustment { kind: Adjust::Borrow(AutoBorrow::Ref(region, mutbl)), target });
-
-            if let Some(unsize_target) = pick.unsize {
-                target = self
-                    .tcx
-                    .mk_ref(region, ty::TypeAndMut { mutbl: mutbl.into(), ty: unsize_target });
-                adjustments.push(Adjustment { kind: Adjust::Pointer(PointerCast::Unsize), target });
+        match &pick.autoref_or_ptr_adjustment {
+            Some(probe::AutorefOrPtrAdjustment::Autoref { mutbl, unsize }) => {
+                let region = self.next_region_var(infer::Autoref(self.span, pick.item));
+                target = self.tcx.mk_ref(region, ty::TypeAndMut { mutbl: *mutbl, ty: target });
+                let mutbl = match mutbl {
+                    hir::Mutability::Not => AutoBorrowMutability::Not,
+                    hir::Mutability::Mut => AutoBorrowMutability::Mut {
+                        // Method call receivers are the primary use case
+                        // for two-phase borrows.
+                        allow_two_phase_borrow: AllowTwoPhase::Yes,
+                    },
+                };
+                adjustments.push(Adjustment {
+                    kind: Adjust::Borrow(AutoBorrow::Ref(region, mutbl)),
+                    target,
+                });
+
+                if let Some(unsize_target) = unsize {
+                    target = self
+                        .tcx
+                        .mk_ref(region, ty::TypeAndMut { mutbl: mutbl.into(), ty: unsize_target });
+                    adjustments
+                        .push(Adjustment { kind: Adjust::Pointer(PointerCast::Unsize), target });
+                }
+            }
+            Some(probe::AutorefOrPtrAdjustment::ToConstPtr) => {
+                target = match target.kind() {
+                    ty::RawPtr(ty::TypeAndMut { ty, mutbl }) => {
+                        assert_eq!(*mutbl, hir::Mutability::Mut);
+                        self.tcx.mk_ptr(ty::TypeAndMut { mutbl: hir::Mutability::Not, ty })
+                    }
+                    other => panic!("Cannot adjust receiver type {:?} to const ptr", other),
+                };
+
+                adjustments.push(Adjustment {
+                    kind: Adjust::Pointer(PointerCast::MutToConstPointer),
+                    target,
+                });
             }
-        } else {
-            // No unsizing should be performed without autoref (at
-            // least during method dispach). This is because we
-            // currently only unsize `[T;N]` to `[T]`, and naturally
-            // that must occur being a reference.
-            assert!(pick.unsize.is_none());
+            None => {}
         }
 
         self.register_predicates(autoderef.into_obligations());
@@ -300,7 +315,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
         // variables.
         let generics = self.tcx.generics_of(pick.item.def_id);
 
-        let arg_count_correct = AstConv::check_generic_arg_count_for_call(
+        let arg_count_correct = <dyn AstConv<'_>>::check_generic_arg_count_for_call(
             self.tcx,
             self.span,
             pick.item.def_id,
@@ -338,12 +353,13 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
             ) -> subst::GenericArg<'tcx> {
                 match (&param.kind, arg) {
                     (GenericParamDefKind::Lifetime, GenericArg::Lifetime(lt)) => {
-                        AstConv::ast_region_to_region(self.cfcx.fcx, lt, Some(param)).into()
+                        <dyn AstConv<'_>>::ast_region_to_region(self.cfcx.fcx, lt, Some(param))
+                            .into()
                     }
                     (GenericParamDefKind::Type { .. }, GenericArg::Type(ty)) => {
                         self.cfcx.to_ty(ty).into()
                     }
-                    (GenericParamDefKind::Const, GenericArg::Const(ct)) => {
+                    (GenericParamDefKind::Const { .. }, GenericArg::Const(ct)) => {
                         self.cfcx.const_arg_to_const(&ct.value, param.def_id).into()
                     }
                     _ => unreachable!(),
@@ -359,13 +375,13 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
                 self.cfcx.var_for_def(self.cfcx.span, param)
             }
         }
-        AstConv::create_substs_for_generic_args(
+        <dyn AstConv<'_>>::create_substs_for_generic_args(
             self.tcx,
             pick.item.def_id,
             parent_substs,
             false,
             None,
-            arg_count_correct,
+            &arg_count_correct,
             &mut MethodSubstsCtxt { cfcx: self, pick, seg },
         )
     }
@@ -481,10 +497,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
             // We don't care about regions here.
             .filter_map(|obligation| match obligation.predicate.kind().skip_binder() {
                 ty::PredicateKind::Trait(trait_pred, _) if trait_pred.def_id() == sized_def_id => {
-                    let span = predicates
-                        .predicates
-                        .iter()
-                        .zip(predicates.spans.iter())
+                    let span = iter::zip(&predicates.predicates, &predicates.spans)
                         .find_map(
                             |(p, span)| {
                                 if *p == obligation.predicate { Some(*span) } else { None }
@@ -537,7 +550,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
         upcast_trait_refs.into_iter().next().unwrap()
     }
 
-    fn replace_bound_vars_with_fresh_vars<T>(&self, value: ty::Binder<T>) -> T
+    fn replace_bound_vars_with_fresh_vars<T>(&self, value: ty::Binder<'tcx, T>) -> T
     where
         T: TypeFoldable<'tcx>,
     {
diff --git a/compiler/rustc_typeck/src/check/method/mod.rs b/compiler/rustc_typeck/src/check/method/mod.rs
index 9a3d1e42b73..bd7ffd057b4 100644
--- a/compiler/rustc_typeck/src/check/method/mod.rs
+++ b/compiler/rustc_typeck/src/check/method/mod.rs
@@ -45,6 +45,7 @@ pub struct MethodCallee<'tcx> {
     pub sig: ty::FnSig<'tcx>,
 }
 
+#[derive(Debug)]
 pub enum MethodError<'tcx> {
     // Did not find an applicable method, but we did find various near-misses that may work.
     NoMatch(NoMatchData<'tcx>),
@@ -66,6 +67,7 @@ pub enum MethodError<'tcx> {
 
 // Contains a list of static methods that may apply, a list of unsatisfied trait predicates which
 // could lead to matches if satisfied, and a list of not-in-scope traits which may work.
+#[derive(Debug)]
 pub struct NoMatchData<'tcx> {
     pub static_candidates: Vec<CandidateSource>,
     pub unsatisfied_predicates: Vec<(ty::Predicate<'tcx>, Option<ty::Predicate<'tcx>>)>,
@@ -308,7 +310,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Construct a trait-reference `self_ty : Trait<input_tys>`
         let substs = InternalSubsts::for_item(self.tcx, trait_def_id, |param, _| {
             match param.kind {
-                GenericParamDefKind::Lifetime | GenericParamDefKind::Const => {}
+                GenericParamDefKind::Lifetime | GenericParamDefKind::Const { .. } => {}
                 GenericParamDefKind::Type { .. } => {
                     if param.index == 0 {
                         return self_ty.into();
@@ -397,7 +399,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         obligations.extend(traits::predicates_for_generics(cause.clone(), self.param_env, bounds));
 
         // Also add an obligation for the method type being well-formed.
-        let method_ty = tcx.mk_fn_ptr(ty::Binder::bind(fn_sig));
+        let method_ty = tcx.mk_fn_ptr(ty::Binder::bind(fn_sig, tcx));
         debug!(
             "lookup_in_trait_adjusted: matched method method_ty={:?} obligation={:?}",
             method_ty, obligation
diff --git a/compiler/rustc_typeck/src/check/method/probe.rs b/compiler/rustc_typeck/src/check/method/probe.rs
index 38ff88553e8..c2998c84131 100644
--- a/compiler/rustc_typeck/src/check/method/probe.rs
+++ b/compiler/rustc_typeck/src/check/method/probe.rs
@@ -83,6 +83,8 @@ struct ProbeContext<'a, 'tcx> {
     unsatisfied_predicates: Vec<(ty::Predicate<'tcx>, Option<ty::Predicate<'tcx>>)>,
 
     is_suggestion: IsSuggestion,
+
+    scope_expr_id: hir::HirId,
 }
 
 impl<'a, 'tcx> Deref for ProbeContext<'a, 'tcx> {
@@ -154,6 +156,42 @@ enum ProbeResult {
     Match,
 }
 
+/// When adjusting a receiver we often want to do one of
+///
+/// - Add a `&` (or `&mut`), converting the receiver from `T` to `&T` (or `&mut T`)
+/// - If the receiver has type `*mut T`, convert it to `*const T`
+///
+/// This type tells us which one to do.
+///
+/// Note that in principle we could do both at the same time. For example, when the receiver has
+/// type `T`, we could autoref it to `&T`, then convert to `*const T`. Or, when it has type `*mut
+/// T`, we could convert it to `*const T`, then autoref to `&*const T`. However, currently we do
+/// (at most) one of these. Either the receiver has type `T` and we convert it to `&T` (or with
+/// `mut`), or it has type `*mut T` and we convert it to `*const T`.
+#[derive(Debug, PartialEq, Clone)]
+pub enum AutorefOrPtrAdjustment<'tcx> {
+    /// Receiver has type `T`, add `&` or `&mut` (it `T` is `mut`), and maybe also "unsize" it.
+    /// Unsizing is used to convert a `[T; N]` to `[T]`, which only makes sense when autorefing.
+    Autoref {
+        mutbl: hir::Mutability,
+
+        /// Indicates that the source expression should be "unsized" to a target type. This should
+        /// probably eventually go away in favor of just coercing method receivers.
+        unsize: Option<Ty<'tcx>>,
+    },
+    /// Receiver has type `*mut T`, convert to `*const T`
+    ToConstPtr,
+}
+
+impl<'tcx> AutorefOrPtrAdjustment<'tcx> {
+    fn get_unsize(&self) -> Option<Ty<'tcx>> {
+        match self {
+            AutorefOrPtrAdjustment::Autoref { mutbl: _, unsize } => unsize.clone(),
+            AutorefOrPtrAdjustment::ToConstPtr => None,
+        }
+    }
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Pick<'tcx> {
     pub item: ty::AssocItem,
@@ -165,17 +203,9 @@ pub struct Pick<'tcx> {
     ///     A = expr | *expr | **expr | ...
     pub autoderefs: usize,
 
-    /// Indicates that an autoref is applied after the optional autoderefs
-    ///
-    ///     B = A | &A | &mut A
-    pub autoref: Option<hir::Mutability>,
-
-    /// Indicates that the source expression should be "unsized" to a
-    /// target type. This should probably eventually go away in favor
-    /// of just coercing method receivers.
-    ///
-    ///     C = B | unsize(B)
-    pub unsize: Option<Ty<'tcx>>,
+    /// Indicates that we want to add an autoref (and maybe also unsize it), or if the receiver is
+    /// `*mut T`, convert it to `*const T`.
+    pub autoref_or_ptr_adjustment: Option<AutorefOrPtrAdjustment<'tcx>>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -420,6 +450,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 orig_values,
                 steps.steps,
                 is_suggestion,
+                scope_expr_id,
             );
 
             probe_cx.assemble_inherent_candidates();
@@ -519,6 +550,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         orig_steps_var_values: OriginalQueryValues<'tcx>,
         steps: Lrc<Vec<CandidateStep<'tcx>>>,
         is_suggestion: IsSuggestion,
+        scope_expr_id: hir::HirId,
     ) -> ProbeContext<'a, 'tcx> {
         ProbeContext {
             fcx,
@@ -536,6 +568,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             private_candidate: None,
             unsatisfied_predicates: Vec::new(),
             is_suggestion,
+            scope_expr_id,
         }
     }
 
@@ -1086,6 +1119,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 self.pick_by_value_method(step, self_ty).or_else(|| {
                     self.pick_autorefd_method(step, self_ty, hir::Mutability::Not)
                         .or_else(|| self.pick_autorefd_method(step, self_ty, hir::Mutability::Mut))
+                        .or_else(|| self.pick_const_ptr_method(step, self_ty))
                 })
             })
             .next()
@@ -1113,7 +1147,10 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 // Insert a `&*` or `&mut *` if this is a reference type:
                 if let ty::Ref(_, _, mutbl) = *step.self_ty.value.value.kind() {
                     pick.autoderefs += 1;
-                    pick.autoref = Some(mutbl);
+                    pick.autoref_or_ptr_adjustment = Some(AutorefOrPtrAdjustment::Autoref {
+                        mutbl,
+                        unsize: pick.autoref_or_ptr_adjustment.and_then(|a| a.get_unsize()),
+                    })
                 }
 
                 pick
@@ -1136,8 +1173,39 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         self.pick_method(autoref_ty).map(|r| {
             r.map(|mut pick| {
                 pick.autoderefs = step.autoderefs;
-                pick.autoref = Some(mutbl);
-                pick.unsize = step.unsize.then_some(self_ty);
+                pick.autoref_or_ptr_adjustment = Some(AutorefOrPtrAdjustment::Autoref {
+                    mutbl,
+                    unsize: step.unsize.then_some(self_ty),
+                });
+                pick
+            })
+        })
+    }
+
+    /// If `self_ty` is `*mut T` then this picks `*const T` methods. The reason why we have a
+    /// special case for this is because going from `*mut T` to `*const T` with autoderefs and
+    /// autorefs would require dereferencing the pointer, which is not safe.
+    fn pick_const_ptr_method(
+        &mut self,
+        step: &CandidateStep<'tcx>,
+        self_ty: Ty<'tcx>,
+    ) -> Option<PickResult<'tcx>> {
+        // Don't convert an unsized reference to ptr
+        if step.unsize {
+            return None;
+        }
+
+        let ty = match self_ty.kind() {
+            ty::RawPtr(ty::TypeAndMut { ty, mutbl: hir::Mutability::Mut }) => ty,
+            _ => return None,
+        };
+
+        let const_self_ty = ty::TypeAndMut { ty, mutbl: hir::Mutability::Not };
+        let const_ptr_ty = self.tcx.mk_ptr(const_self_ty);
+        self.pick_method(const_ptr_ty).map(|r| {
+            r.map(|mut pick| {
+                pick.autoderefs = step.autoderefs;
+                pick.autoref_or_ptr_adjustment = Some(AutorefOrPtrAdjustment::ToConstPtr);
                 pick
             })
         })
@@ -1249,7 +1317,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     ) {
         self.tcx.struct_span_lint_hir(
             lint::builtin::UNSTABLE_NAME_COLLISIONS,
-            self.fcx.body_id,
+            self.scope_expr_id,
             self.span,
             |lint| {
                 let def_kind = stable_pick.item.kind.as_def_kind();
@@ -1510,8 +1578,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             kind: TraitPick,
             import_ids: probes[0].0.import_ids.clone(),
             autoderefs: 0,
-            autoref: None,
-            unsize: None,
+            autoref_or_ptr_adjustment: None,
         })
     }
 
@@ -1532,6 +1599,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 self.orig_steps_var_values.clone(),
                 steps,
                 IsSuggestion(true),
+                self.scope_expr_id,
             );
             pcx.allow_similar_names = true;
             pcx.assemble_inherent_candidates();
@@ -1638,7 +1706,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                             // In general, during probe we erase regions.
                             self.tcx.lifetimes.re_erased.into()
                         }
-                        GenericParamDefKind::Type { .. } | GenericParamDefKind::Const => {
+                        GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
                             self.var_for_def(self.span, param)
                         }
                     }
@@ -1691,7 +1759,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     ///    region got replaced with the same variable, which requires a bit more coordination
     ///    and/or tracking the substitution and
     ///    so forth.
-    fn erase_late_bound_regions<T>(&self, value: ty::Binder<T>) -> T
+    fn erase_late_bound_regions<T>(&self, value: ty::Binder<'tcx, T>) -> T
     where
         T: TypeFoldable<'tcx>,
     {
@@ -1700,7 +1768,9 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
 
     /// Finds the method with the appropriate name (or return type, as the case may be). If
     /// `allow_similar_names` is set, find methods with close-matching names.
-    fn impl_or_trait_item(&self, def_id: DefId) -> Vec<ty::AssocItem> {
+    // The length of the returned iterator is nearly always 0 or 1 and this
+    // method is fairly hot.
+    fn impl_or_trait_item(&self, def_id: DefId) -> SmallVec<[ty::AssocItem; 1]> {
         if let Some(name) = self.method_name {
             if self.allow_similar_names {
                 let max_dist = max(name.as_str().len(), 3) / 3;
@@ -1716,7 +1786,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             } else {
                 self.fcx
                     .associated_item(def_id, name, Namespace::ValueNS)
-                    .map_or_else(Vec::new, |x| vec![x])
+                    .map_or_else(SmallVec::new, |x| SmallVec::from_buf([x]))
             }
         } else {
             self.tcx.associated_items(def_id).in_definition_order().copied().collect()
@@ -1748,8 +1818,7 @@ impl<'tcx> Candidate<'tcx> {
             },
             import_ids: self.import_ids.clone(),
             autoderefs: 0,
-            autoref: None,
-            unsize: None,
+            autoref_or_ptr_adjustment: None,
         }
     }
 }
diff --git a/compiler/rustc_typeck/src/check/method/suggest.rs b/compiler/rustc_typeck/src/check/method/suggest.rs
index e6c551ff4d4..02fe8312c4c 100644
--- a/compiler/rustc_typeck/src/check/method/suggest.rs
+++ b/compiler/rustc_typeck/src/check/method/suggest.rs
@@ -68,12 +68,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    pub fn report_method_error<'b>(
+    pub fn report_method_error(
         &self,
         span: Span,
         rcvr_ty: Ty<'tcx>,
         item_name: Ident,
-        source: SelfSource<'b>,
+        source: SelfSource<'tcx>,
         error: MethodError<'tcx>,
         args: Option<&'tcx [hir::Expr<'tcx>]>,
     ) -> Option<DiagnosticBuilder<'_>> {
@@ -323,8 +323,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 err.span_suggestion(
                                     lit.span,
                                     &format!(
-                                        "you must specify a concrete type for \
-                                              this numeric value, like `{}`",
+                                        "you must specify a concrete type for this numeric value, \
+                                         like `{}`",
                                         concrete_type
                                     ),
                                     format!("{}_{}", snippet, concrete_type),
@@ -975,17 +975,79 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    fn suggest_traits_to_import<'b>(
+    fn suggest_traits_to_import(
         &self,
         err: &mut DiagnosticBuilder<'_>,
         span: Span,
         rcvr_ty: Ty<'tcx>,
         item_name: Ident,
-        source: SelfSource<'b>,
+        source: SelfSource<'tcx>,
         valid_out_of_scope_traits: Vec<DefId>,
         unsatisfied_predicates: &[(ty::Predicate<'tcx>, Option<ty::Predicate<'tcx>>)],
     ) {
-        if self.suggest_valid_traits(err, valid_out_of_scope_traits) {
+        let mut alt_rcvr_sugg = false;
+        if let SelfSource::MethodCall(rcvr) = source {
+            debug!(?span, ?item_name, ?rcvr_ty, ?rcvr);
+            // Try alternative arbitrary self types that could fulfill this call.
+            // FIXME: probe for all types that *could* be arbitrary self-types, not
+            // just this list.
+            for (rcvr_ty, post) in &[
+                (rcvr_ty, ""),
+                (self.tcx.mk_mut_ref(&ty::ReErased, rcvr_ty), "&mut "),
+                (self.tcx.mk_imm_ref(&ty::ReErased, rcvr_ty), "&"),
+            ] {
+                for (rcvr_ty, pre) in &[
+                    (self.tcx.mk_lang_item(rcvr_ty, LangItem::OwnedBox), "Box::new"),
+                    (self.tcx.mk_lang_item(rcvr_ty, LangItem::Pin), "Pin::new"),
+                    (self.tcx.mk_diagnostic_item(rcvr_ty, sym::Arc), "Arc::new"),
+                    (self.tcx.mk_diagnostic_item(rcvr_ty, sym::Rc), "Rc::new"),
+                ] {
+                    if let Some(new_rcvr_t) = *rcvr_ty {
+                        if let Ok(pick) = self.lookup_probe(
+                            span,
+                            item_name,
+                            new_rcvr_t,
+                            rcvr,
+                            crate::check::method::probe::ProbeScope::AllTraits,
+                        ) {
+                            debug!("try_alt_rcvr: pick candidate {:?}", pick);
+                            let did = Some(pick.item.container.id());
+                            // We don't want to suggest a container type when the missing
+                            // method is `.clone()` or `.deref()` otherwise we'd suggest
+                            // `Arc::new(foo).clone()`, which is far from what the user wants.
+                            let skip = [
+                                self.tcx.lang_items().clone_trait(),
+                                self.tcx.lang_items().deref_trait(),
+                                self.tcx.lang_items().deref_mut_trait(),
+                                self.tcx.lang_items().drop_trait(),
+                            ]
+                            .contains(&did);
+                            // Make sure the method is defined for the *actual* receiver: we don't
+                            // want to treat `Box<Self>` as a receiver if it only works because of
+                            // an autoderef to `&self`
+                            if pick.autoderefs == 0 && !skip {
+                                err.span_label(
+                                    pick.item.ident.span,
+                                    &format!("the method is available for `{}` here", new_rcvr_t),
+                                );
+                                err.multipart_suggestion(
+                                    "consider wrapping the receiver expression with the \
+                                        appropriate type",
+                                    vec![
+                                        (rcvr.span.shrink_to_lo(), format!("{}({}", pre, post)),
+                                        (rcvr.span.shrink_to_hi(), ")".to_string()),
+                                    ],
+                                    Applicability::MaybeIncorrect,
+                                );
+                                // We don't care about the other suggestions.
+                                alt_rcvr_sugg = true;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        if !alt_rcvr_sugg && self.suggest_valid_traits(err, valid_out_of_scope_traits) {
             return;
         }
 
@@ -1075,6 +1137,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 "the method might not be found because of this arbitrary self type",
             );
         }
+        if alt_rcvr_sugg {
+            return;
+        }
 
         if !candidates.is_empty() {
             // Sort from most relevant to least relevant.
@@ -1284,7 +1349,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     /// Checks whether there is a local type somewhere in the chain of
     /// autoderefs of `rcvr_ty`.
-    fn type_derefs_to_local(&self, span: Span, rcvr_ty: Ty<'tcx>, source: SelfSource<'_>) -> bool {
+    fn type_derefs_to_local(
+        &self,
+        span: Span,
+        rcvr_ty: Ty<'tcx>,
+        source: SelfSource<'tcx>,
+    ) -> bool {
         fn is_local(ty: Ty<'_>) -> bool {
             match ty.kind() {
                 ty::Adt(def, _) => def.did.is_local(),
@@ -1310,7 +1380,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 }
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub enum SelfSource<'a> {
     QPath(&'a hir::Ty<'a>),
     MethodCall(&'a hir::Expr<'a> /* rcvr */),
@@ -1466,11 +1536,12 @@ impl intravisit::Visitor<'tcx> for UsePlacementFinder<'tcx> {
                     if self.span.map_or(true, |span| item.span < span) {
                         if !item.span.from_expansion() {
                             // Don't insert between attributes and an item.
-                            if item.attrs.is_empty() {
+                            let attrs = self.tcx.hir().attrs(item.hir_id());
+                            if attrs.is_empty() {
                                 self.span = Some(item.span.shrink_to_lo());
                             } else {
                                 // Find the first attribute on the item.
-                                for attr in item.attrs {
+                                for attr in attrs {
                                     if self.span.map_or(true, |span| attr.span < span) {
                                         self.span = Some(attr.span.shrink_to_lo());
                                     }
diff --git a/compiler/rustc_typeck/src/check/mod.rs b/compiler/rustc_typeck/src/check/mod.rs
index bb85336d7fb..80e173de6b6 100644
--- a/compiler/rustc_typeck/src/check/mod.rs
+++ b/compiler/rustc_typeck/src/check/mod.rs
@@ -121,9 +121,9 @@ use rustc_middle::ty::{self, RegionKind, Ty, TyCtxt, UserType};
 use rustc_session::config;
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
-use rustc_span::source_map::DUMMY_SP;
 use rustc_span::symbol::{kw, Ident};
 use rustc_span::{self, BytePos, MultiSpan, Span};
+use rustc_span::{source_map::DUMMY_SP, sym};
 use rustc_target::abi::VariantIdx;
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits;
@@ -495,8 +495,9 @@ fn typeck_with_fallback<'tcx>(
         let fcx = if let (Some(header), Some(decl)) = (fn_header, fn_decl) {
             let fn_sig = if crate::collect::get_infer_ret_ty(&decl.output).is_some() {
                 let fcx = FnCtxt::new(&inh, param_env, body.value.hir_id);
-                AstConv::ty_of_fn(
+                <dyn AstConv<'_>>::ty_of_fn(
                     &fcx,
+                    id,
                     header.unsafety,
                     header.abi,
                     decl,
@@ -527,7 +528,7 @@ fn typeck_with_fallback<'tcx>(
             let fcx = FnCtxt::new(&inh, param_env, body.value.hir_id);
             let expected_type = body_ty
                 .and_then(|ty| match ty.kind {
-                    hir::TyKind::Infer => Some(AstConv::ast_ty_to_ty(&fcx, ty)),
+                    hir::TyKind::Infer => Some(<dyn AstConv<'_>>::ast_ty_to_ty(&fcx, ty)),
                     _ => None,
                 })
                 .unwrap_or_else(|| match tcx.hir().get(id) {
@@ -539,6 +540,19 @@ fn typeck_with_fallback<'tcx>(
                             kind: TypeVariableOriginKind::TypeInference,
                             span,
                         }),
+                        Node::Expr(&hir::Expr { kind: hir::ExprKind::InlineAsm(ia), .. })
+                            if ia.operands.iter().any(|(op, _op_sp)| match op {
+                                hir::InlineAsmOperand::Const { anon_const } => {
+                                    anon_const.hir_id == id
+                                }
+                                _ => false,
+                            }) =>
+                        {
+                            fcx.next_ty_var(TypeVariableOrigin {
+                                kind: TypeVariableOriginKind::MiscVariable,
+                                span,
+                            })
+                        }
                         _ => fallback(),
                     },
                     _ => fallback(),
@@ -547,11 +561,12 @@ fn typeck_with_fallback<'tcx>(
             let expected_type = fcx.normalize_associated_types_in(body.value.span, expected_type);
             fcx.require_type_is_sized(expected_type, body.value.span, traits::ConstSized);
 
-            let revealed_ty = if tcx.features().impl_trait_in_bindings {
-                fcx.instantiate_opaque_types_from_value(id, expected_type, body.value.span)
-            } else {
-                expected_type
-            };
+            let revealed_ty = fcx.instantiate_opaque_types_from_value(
+                id,
+                expected_type,
+                body.value.span,
+                Some(sym::impl_trait_in_bindings),
+            );
 
             // Gather locals in statics (because of block expressions).
             GatherLocalsVisitor::new(&fcx, id).visit_body(body);
diff --git a/compiler/rustc_typeck/src/check/pat.rs b/compiler/rustc_typeck/src/check/pat.rs
index f8ca916caf1..53593b9bab4 100644
--- a/compiler/rustc_typeck/src/check/pat.rs
+++ b/compiler/rustc_typeck/src/check/pat.rs
@@ -680,7 +680,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         pat: &'tcx Pat<'tcx>,
         qpath: &hir::QPath<'_>,
-        fields: &'tcx [hir::FieldPat<'tcx>],
+        fields: &'tcx [hir::PatField<'tcx>],
         etc: bool,
         expected: Ty<'tcx>,
         def_bm: BindingMode,
@@ -861,7 +861,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn check_pat_tuple_struct(
         &self,
         pat: &'tcx Pat<'tcx>,
-        qpath: &hir::QPath<'_>,
+        qpath: &'tcx hir::QPath<'tcx>,
         subpats: &'tcx [&'tcx Pat<'tcx>],
         ddpos: Option<usize>,
         expected: Ty<'tcx>,
@@ -1151,7 +1151,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         adt_ty: Ty<'tcx>,
         pat: &'tcx Pat<'tcx>,
         variant: &'tcx ty::VariantDef,
-        fields: &'tcx [hir::FieldPat<'tcx>],
+        fields: &'tcx [hir::PatField<'tcx>],
         etc: bool,
         def_bm: BindingMode,
         ti: TopInfo<'tcx>,
@@ -1291,7 +1291,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         variant: &VariantDef,
         pat: &'_ Pat<'_>,
-        fields: &[hir::FieldPat<'_>],
+        fields: &[hir::PatField<'_>],
     ) -> Option<DiagnosticBuilder<'_>> {
         // if this is a tuple struct, then all field names will be numbers
         // so if any fields in a struct pattern use shorthand syntax, they will
@@ -1446,7 +1446,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn error_tuple_variant_as_struct_pat(
         &self,
         pat: &Pat<'_>,
-        fields: &'tcx [hir::FieldPat<'tcx>],
+        fields: &'tcx [hir::PatField<'tcx>],
         variant: &ty::VariantDef,
     ) -> Option<DiagnosticBuilder<'tcx>> {
         if let (CtorKind::Fn, PatKind::Struct(qpath, ..)) = (variant.ctor_kind, &pat.kind) {
@@ -1484,7 +1484,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     fn get_suggested_tuple_struct_pattern(
         &self,
-        fields: &[hir::FieldPat<'_>],
+        fields: &[hir::PatField<'_>],
         variant: &VariantDef,
     ) -> String {
         let variant_field_idents = variant.fields.iter().map(|f| f.ident).collect::<Vec<Ident>>();
@@ -1528,7 +1528,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn error_no_accessible_fields(
         &self,
         pat: &Pat<'_>,
-        fields: &'tcx [hir::FieldPat<'tcx>],
+        fields: &'tcx [hir::PatField<'tcx>],
     ) -> DiagnosticBuilder<'tcx> {
         let mut err = self
             .tcx
@@ -1574,7 +1574,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         pat: &Pat<'_>,
         unmentioned_fields: &[(&ty::FieldDef, Ident)],
-        fields: &'tcx [hir::FieldPat<'tcx>],
+        fields: &'tcx [hir::PatField<'tcx>],
     ) -> DiagnosticBuilder<'tcx> {
         let field_names = if unmentioned_fields.len() == 1 {
             format!("field `{}`", unmentioned_fields[0].1)
diff --git a/compiler/rustc_typeck/src/check/place_op.rs b/compiler/rustc_typeck/src/check/place_op.rs
index 254e41706f9..5bd385107ca 100644
--- a/compiler/rustc_typeck/src/check/place_op.rs
+++ b/compiler/rustc_typeck/src/check/place_op.rs
@@ -103,9 +103,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let method =
                 self.try_overloaded_place_op(expr.span, self_ty, &[input_ty], PlaceOp::Index);
 
-            let result = method.map(|ok| {
+            if let Some(result) = method {
                 debug!("try_index_step: success, using overloaded indexing");
-                let method = self.register_infer_ok_obligations(ok);
+                let method = self.register_infer_ok_obligations(result);
 
                 let mut adjustments = self.adjust_steps(autoderef);
                 if let ty::Ref(region, _, hir::Mutability::Not) = method.sig.inputs()[0].kind() {
@@ -128,10 +128,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.apply_adjustments(base_expr, adjustments);
 
                 self.write_method_call(expr.hir_id, method);
-                (input_ty, self.make_overloaded_place_return_type(method).ty)
-            });
-            if result.is_some() {
-                return result;
+
+                return Some((input_ty, self.make_overloaded_place_return_type(method).ty));
             }
         }
 
diff --git a/compiler/rustc_typeck/src/check/regionck.rs b/compiler/rustc_typeck/src/check/regionck.rs
index 88e8dd3cb12..8f8514cadb7 100644
--- a/compiler/rustc_typeck/src/check/regionck.rs
+++ b/compiler/rustc_typeck/src/check/regionck.rs
@@ -354,7 +354,7 @@ impl<'a, 'tcx> Visitor<'tcx> for RegionCtxt<'a, 'tcx> {
         hir_id: hir::HirId,
     ) {
         assert!(
-            matches!(fk, intravisit::FnKind::Closure(..)),
+            matches!(fk, intravisit::FnKind::Closure),
             "visit_fn invoked for something other than a closure"
         );
 
@@ -771,21 +771,39 @@ impl<'a, 'tcx> RegionCtxt<'a, 'tcx> {
         debug!("link_upvar_region(borrorw_region={:?}, upvar_id={:?}", borrow_region, upvar_id);
         // A by-reference upvar can't be borrowed for longer than the
         // upvar is borrowed from the environment.
-        match self.typeck_results.borrow().upvar_capture(upvar_id) {
-            ty::UpvarCapture::ByRef(upvar_borrow) => {
-                self.sub_regions(
-                    infer::ReborrowUpvar(span, upvar_id),
-                    borrow_region,
-                    upvar_borrow.region,
-                );
-                if let ty::ImmBorrow = upvar_borrow.kind {
-                    debug!("link_upvar_region: capture by shared ref");
-                    return;
+        let closure_local_def_id = upvar_id.closure_expr_id;
+        let mut all_captures_are_imm_borrow = true;
+        for captured_place in self
+            .typeck_results
+            .borrow()
+            .closure_min_captures
+            .get(&closure_local_def_id.to_def_id())
+            .and_then(|root_var_min_cap| root_var_min_cap.get(&upvar_id.var_path.hir_id))
+            .into_iter()
+            .flatten()
+        {
+            match captured_place.info.capture_kind {
+                ty::UpvarCapture::ByRef(upvar_borrow) => {
+                    self.sub_regions(
+                        infer::ReborrowUpvar(span, upvar_id),
+                        borrow_region,
+                        upvar_borrow.region,
+                    );
+                    if let ty::ImmBorrow = upvar_borrow.kind {
+                        debug!("link_upvar_region: capture by shared ref");
+                    } else {
+                        all_captures_are_imm_borrow = false;
+                    }
+                }
+                ty::UpvarCapture::ByValue(_) => {
+                    all_captures_are_imm_borrow = false;
                 }
             }
-            ty::UpvarCapture::ByValue(_) => {}
         }
-        let fn_hir_id = self.tcx.hir().local_def_id_to_hir_id(upvar_id.closure_expr_id);
+        if all_captures_are_imm_borrow {
+            return;
+        }
+        let fn_hir_id = self.tcx.hir().local_def_id_to_hir_id(closure_local_def_id);
         let ty = self.resolve_node_type(fn_hir_id);
         debug!("link_upvar_region: ty={:?}", ty);
 
diff --git a/compiler/rustc_typeck/src/check/upvar.rs b/compiler/rustc_typeck/src/check/upvar.rs
index 69c09528662..751eebb9f95 100644
--- a/compiler/rustc_typeck/src/check/upvar.rs
+++ b/compiler/rustc_typeck/src/check/upvar.rs
@@ -30,18 +30,18 @@
 //! then mean that all later passes would have to check for these figments
 //! and report an error, and it just seems like more mess in the end.)
 
-use super::writeback::Resolver;
 use super::FnCtxt;
 
 use crate::expr_use_visitor as euv;
 use rustc_data_structures::fx::FxIndexMap;
+use rustc_errors::Applicability;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_infer::infer::UpvarRegion;
 use rustc_middle::hir::place::{Place, PlaceBase, PlaceWithHirId, Projection, ProjectionKind};
-use rustc_middle::ty::fold::TypeFoldable;
+use rustc_middle::mir::FakeReadCause;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeckResults, UpvarSubsts};
 use rustc_session::lint;
 use rustc_span::sym;
@@ -50,6 +50,8 @@ use rustc_span::{MultiSpan, Span, Symbol};
 use rustc_index::vec::Idx;
 use rustc_target::abi::VariantIdx;
 
+use std::iter;
+
 /// Describe the relationship between the paths of two places
 /// eg:
 /// - `foo` is ancestor of `foo.bar.baz`
@@ -90,7 +92,7 @@ impl<'a, 'tcx> Visitor<'tcx> for InferBorrowKindVisitor<'a, 'tcx> {
         if let hir::ExprKind::Closure(cc, _, body_id, _, _) = expr.kind {
             let body = self.fcx.tcx.hir().body(body_id);
             self.visit_body(body);
-            self.fcx.analyze_closure(expr.hir_id, expr.span, body, cc);
+            self.fcx.analyze_closure(expr.hir_id, expr.span, body_id, body, cc);
         }
 
         intravisit::walk_expr(self, expr);
@@ -103,6 +105,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         closure_hir_id: hir::HirId,
         span: Span,
+        body_id: hir::BodyId,
         body: &'tcx hir::Body<'tcx>,
         capture_clause: hir::CaptureBy,
     ) {
@@ -145,6 +148,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             current_closure_kind: ty::ClosureKind::LATTICE_BOTTOM,
             current_origin: None,
             capture_information: Default::default(),
+            fake_reads: Default::default(),
         };
         euv::ExprUseVisitor::new(
             &mut delegate,
@@ -165,7 +169,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let closure_hir_id = self.tcx.hir().local_def_id_to_hir_id(local_def_id);
         if should_do_migration_analysis(self.tcx, closure_hir_id) {
-            self.perform_2229_migration_anaysis(closure_def_id, capture_clause, span, body);
+            self.perform_2229_migration_anaysis(closure_def_id, body_id, capture_clause, span);
         }
 
         // We now fake capture information for all variables that are mentioned within the closure
@@ -220,8 +224,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         self.log_closure_min_capture_info(closure_def_id, span);
 
-        self.min_captures_to_closure_captures_bridge(closure_def_id);
-
         // Now that we've analyzed the closure, we know how each
         // variable is borrowed, and we know what traits the closure
         // implements (Fn vs FnMut etc). We now have some updates to do
@@ -246,6 +248,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let final_tupled_upvars_type = self.tcx.mk_tup(final_upvar_tys.iter());
         self.demand_suptype(span, substs.tupled_upvars_ty(), final_tupled_upvars_type);
 
+        let fake_reads = delegate
+            .fake_reads
+            .into_iter()
+            .map(|(place, cause, hir_id)| (place, cause, hir_id))
+            .collect();
+        self.typeck_results.borrow_mut().closure_fake_reads.insert(closure_def_id, fake_reads);
+
         // If we are also inferred the closure kind here,
         // process any deferred resolutions.
         let deferred_call_resolutions = self.remove_deferred_call_resolutions(closure_def_id);
@@ -284,80 +293,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .collect()
     }
 
-    /// Bridge for closure analysis
-    /// ----------------------------
-    ///
-    /// For closure with DefId `c`, the bridge converts structures required for supporting RFC 2229,
-    /// to structures currently used in the compiler for handling closure captures.
-    ///
-    /// For example the following structure will be converted:
-    ///
-    /// closure_min_captures
-    /// foo -> [ {foo.x, ImmBorrow}, {foo.y, MutBorrow} ]
-    /// bar -> [ {bar.z, ByValue}, {bar.q, MutBorrow} ]
-    ///
-    /// to
-    ///
-    /// 1. closure_captures
-    /// foo -> UpvarId(foo, c), bar -> UpvarId(bar, c)
-    ///
-    /// 2. upvar_capture_map
-    /// UpvarId(foo,c) -> MutBorrow, UpvarId(bar, c) -> ByValue
-    fn min_captures_to_closure_captures_bridge(&self, closure_def_id: DefId) {
-        let mut closure_captures: FxIndexMap<hir::HirId, ty::UpvarId> = Default::default();
-        let mut upvar_capture_map = ty::UpvarCaptureMap::default();
-
-        if let Some(min_captures) =
-            self.typeck_results.borrow().closure_min_captures.get(&closure_def_id)
-        {
-            for (var_hir_id, min_list) in min_captures.iter() {
-                for captured_place in min_list {
-                    let place = &captured_place.place;
-                    let capture_info = captured_place.info;
-
-                    let upvar_id = match place.base {
-                        PlaceBase::Upvar(upvar_id) => upvar_id,
-                        base => bug!("Expected upvar, found={:?}", base),
-                    };
-
-                    assert_eq!(upvar_id.var_path.hir_id, *var_hir_id);
-                    assert_eq!(upvar_id.closure_expr_id, closure_def_id.expect_local());
-
-                    closure_captures.insert(*var_hir_id, upvar_id);
-
-                    let new_capture_kind =
-                        if let Some(capture_kind) = upvar_capture_map.get(&upvar_id) {
-                            // upvar_capture_map only stores the UpvarCapture (CaptureKind),
-                            // so we create a fake capture info with no expression.
-                            let fake_capture_info = ty::CaptureInfo {
-                                capture_kind_expr_id: None,
-                                path_expr_id: None,
-                                capture_kind: *capture_kind,
-                            };
-                            determine_capture_info(fake_capture_info, capture_info).capture_kind
-                        } else {
-                            capture_info.capture_kind
-                        };
-                    upvar_capture_map.insert(upvar_id, new_capture_kind);
-                }
-            }
-        }
-        debug!("For closure_def_id={:?}, closure_captures={:#?}", closure_def_id, closure_captures);
-        debug!(
-            "For closure_def_id={:?}, upvar_capture_map={:#?}",
-            closure_def_id, upvar_capture_map
-        );
-
-        if !closure_captures.is_empty() {
-            self.typeck_results
-                .borrow_mut()
-                .closure_captures
-                .insert(closure_def_id, closure_captures);
-
-            self.typeck_results.borrow_mut().upvar_capture_map.extend(upvar_capture_map);
-        }
-    }
-
     /// Analyzes the information collected by `InferBorrowKind` to compute the min number of
     /// Places (and corresponding capture kind) that we need to keep track of to support all
     /// the required captured paths.
@@ -532,20 +467,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn perform_2229_migration_anaysis(
         &self,
         closure_def_id: DefId,
+        body_id: hir::BodyId,
         capture_clause: hir::CaptureBy,
         span: Span,
-        body: &'tcx hir::Body<'tcx>,
     ) {
         let need_migrations = self.compute_2229_migrations(
             closure_def_id,
             span,
             capture_clause,
-            body,
             self.typeck_results.borrow().closure_min_captures.get(&closure_def_id),
         );
 
         if !need_migrations.is_empty() {
-            let migrations_text = migration_suggestion_for_2229(self.tcx, &need_migrations);
+            let (migration_string, migrated_variables_concat) =
+                migration_suggestion_for_2229(self.tcx, &need_migrations);
 
             let local_def_id = closure_def_id.expect_local();
             let closure_hir_id = self.tcx.hir().local_def_id_to_hir_id(local_def_id);
@@ -557,7 +492,35 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let mut diagnostics_builder = lint.build(
                         "drop order affected for closure because of `capture_disjoint_fields`",
                     );
-                    diagnostics_builder.note(&migrations_text);
+                    let closure_body_span = self.tcx.hir().span(body_id.hir_id);
+                    let (sugg, app) =
+                        match self.tcx.sess.source_map().span_to_snippet(closure_body_span) {
+                            Ok(s) => {
+                                let trimmed = s.trim_start();
+
+                                // If the closure contains a block then replace the opening brace
+                                // with "{ let _ = (..); "
+                                let sugg = if let Some('{') = trimmed.chars().next() {
+                                    format!("{{ {}; {}", migration_string, &trimmed[1..])
+                                } else {
+                                    format!("{{ {}; {} }}", migration_string, s)
+                                };
+                                (sugg, Applicability::MachineApplicable)
+                            }
+                            Err(_) => (migration_string.clone(), Applicability::HasPlaceholders),
+                        };
+
+                    let diagnostic_msg = format!(
+                        "add a dummy let to cause {} to be fully captured",
+                        migrated_variables_concat
+                    );
+
+                    diagnostics_builder.span_suggestion(
+                        closure_body_span,
+                        &diagnostic_msg,
+                        sugg,
+                        app,
+                    );
                     diagnostics_builder.emit();
                 },
             );
@@ -578,19 +541,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         closure_def_id: DefId,
         closure_span: Span,
         closure_clause: hir::CaptureBy,
-        body: &'tcx hir::Body<'tcx>,
         min_captures: Option<&ty::RootVariableMinCaptureList<'tcx>>,
     ) -> Vec<hir::HirId> {
-        fn resolve_ty<T: TypeFoldable<'tcx>>(
-            fcx: &FnCtxt<'_, 'tcx>,
-            span: Span,
-            body: &'tcx hir::Body<'tcx>,
-            ty: T,
-        ) -> T {
-            let mut resolver = Resolver::new(fcx, &span, body);
-            ty.fold_with(&mut resolver)
-        }
-
         let upvars = if let Some(upvars) = self.tcx.upvars_mentioned(closure_def_id) {
             upvars
         } else {
@@ -600,7 +552,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let mut need_migrations = Vec::new();
 
         for (&var_hir_id, _) in upvars.iter() {
-            let ty = resolve_ty(self, closure_span, body, self.node_ty(var_hir_id));
+            let ty = self.infcx.resolve_vars_if_possible(self.node_ty(var_hir_id));
 
             if !ty.needs_drop(self.tcx, self.tcx.param_env(closure_def_id.expect_local())) {
                 continue;
@@ -701,7 +653,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// `w[c]`.
     /// Notation:
     /// - Ty(place): Type of place
-    /// - `(a, b)`: Represents the function parameters `base_path_ty` and `captured_projs`
+    /// - `(a, b)`: Represents the function parameters `base_path_ty` and `captured_by_move_projs`
     /// respectively.
     /// ```
     ///                  (Ty(w), [ &[p, x], &[c] ])
@@ -762,7 +714,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         closure_def_id: DefId,
         closure_span: Span,
         base_path_ty: Ty<'tcx>,
-        captured_projs: Vec<&[Projection<'tcx>]>,
+        captured_by_move_projs: Vec<&[Projection<'tcx>]>,
     ) -> bool {
         let needs_drop = |ty: Ty<'tcx>| {
             ty.needs_drop(self.tcx, self.tcx.param_env(closure_def_id.expect_local()))
@@ -787,9 +739,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         //
         // eg. If `a.b` is captured and we are processing `a.b`, then we can't have the closure also
         //     capture `a.b.c`, because that voilates min capture.
-        let is_completely_captured = captured_projs.iter().any(|projs| projs.is_empty());
+        let is_completely_captured = captured_by_move_projs.iter().any(|projs| projs.is_empty());
 
-        assert!(!is_completely_captured || (captured_projs.len() == 1));
+        assert!(!is_completely_captured || (captured_by_move_projs.len() == 1));
 
         if is_completely_captured {
             // The place is captured entirely, so doesn't matter if needs dtor, it will be drop
@@ -797,23 +749,27 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return false;
         }
 
+        if captured_by_move_projs.is_empty() {
+            return needs_drop(base_path_ty);
+        }
+
         if is_drop_defined_for_ty {
             // If drop is implemented for this type then we need it to be fully captured,
-            // which we know it is not because of the previous check. Therefore we need to
-            // do migrate.
-            return true;
-        }
+            // and we know it is not completely captured because of the previous checks.
 
-        if captured_projs.is_empty() {
-            return needs_drop(base_path_ty);
+            // Note that this is a bug in the user code that will be reported by the
+            // borrow checker, since we can't move out of drop types.
+
+            // The bug exists in the user's code pre-migration, and we don't migrate here.
+            return false;
         }
 
         match base_path_ty.kind() {
             // Observations:
-            // - `captured_projs` is not empty. Therefore we can call
-            //   `captured_projs.first().unwrap()` safely.
-            // - All entries in `captured_projs` have atleast one projection.
-            //   Therefore we can call `captured_projs.first().unwrap().first().unwrap()` safely.
+            // - `captured_by_move_projs` is not empty. Therefore we can call
+            //   `captured_by_move_projs.first().unwrap()` safely.
+            // - All entries in `captured_by_move_projs` have atleast one projection.
+            //   Therefore we can call `captured_by_move_projs.first().unwrap().first().unwrap()` safely.
 
             // We don't capture derefs in case of move captures, which would have be applied to
             // access any further paths.
@@ -823,19 +779,19 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             ty::Adt(def, substs) => {
                 // Multi-varaint enums are captured in entirety,
-                // which would've been handled in the case of single empty slice in `captured_projs`.
+                // which would've been handled in the case of single empty slice in `captured_by_move_projs`.
                 assert_eq!(def.variants.len(), 1);
 
                 // Only Field projections can be applied to a non-box Adt.
                 assert!(
-                    captured_projs.iter().all(|projs| matches!(
+                    captured_by_move_projs.iter().all(|projs| matches!(
                         projs.first().unwrap().kind,
                         ProjectionKind::Field(..)
                     ))
                 );
                 def.variants.get(VariantIdx::new(0)).unwrap().fields.iter().enumerate().any(
                     |(i, field)| {
-                        let paths_using_field = captured_projs
+                        let paths_using_field = captured_by_move_projs
                             .iter()
                             .filter_map(|projs| {
                                 if let ProjectionKind::Field(field_idx, _) =
@@ -862,14 +818,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ty::Tuple(..) => {
                 // Only Field projections can be applied to a tuple.
                 assert!(
-                    captured_projs.iter().all(|projs| matches!(
+                    captured_by_move_projs.iter().all(|projs| matches!(
                         projs.first().unwrap().kind,
                         ProjectionKind::Field(..)
                     ))
                 );
 
                 base_path_ty.tuple_fields().enumerate().any(|(i, element_ty)| {
-                    let paths_using_field = captured_projs
+                    let paths_using_field = captured_by_move_projs
                         .iter()
                         .filter_map(|projs| {
                             if let ProjectionKind::Field(field_idx, _) = projs.first().unwrap().kind
@@ -1053,6 +1009,52 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 }
 
+/// Truncate the capture so that the place being borrowed is in accordance with RFC 1240,
+/// which states that it's unsafe to take a reference into a struct marked `repr(packed)`.
+fn restrict_repr_packed_field_ref_capture<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    place: &Place<'tcx>,
+) -> Place<'tcx> {
+    let pos = place.projections.iter().enumerate().position(|(i, p)| {
+        let ty = place.ty_before_projection(i);
+
+        // Return true for fields of packed structs, unless those fields have alignment 1.
+        match p.kind {
+            ProjectionKind::Field(..) => match ty.kind() {
+                ty::Adt(def, _) if def.repr.packed() => {
+                    match tcx.layout_raw(param_env.and(p.ty)) {
+                        Ok(layout) if layout.align.abi.bytes() == 1 => {
+                            // if the alignment is 1, the type can't be further
+                            // disaligned.
+                            debug!(
+                                "restrict_repr_packed_field_ref_capture: ({:?}) - align = 1",
+                                place
+                            );
+                            false
+                        }
+                        _ => {
+                            debug!("restrict_repr_packed_field_ref_capture: ({:?}) - true", place);
+                            true
+                        }
+                    }
+                }
+
+                _ => false,
+            },
+            _ => false,
+        }
+    });
+
+    let mut place = place.clone();
+
+    if let Some(pos) = pos {
+        place.projections.truncate(pos);
+    }
+
+    place
+}
+
 struct InferBorrowKind<'a, 'tcx> {
     fcx: &'a FnCtxt<'a, 'tcx>,
 
@@ -1102,6 +1104,7 @@ struct InferBorrowKind<'a, 'tcx> {
     /// Place { V1, [ProjectionKind::Field(Index=1, Variant=0)] } : CaptureKind { E2, MutableBorrow }
     /// ```
     capture_information: InferredCaptureInformation<'tcx>,
+    fake_reads: Vec<(Place<'tcx>, FakeReadCause, hir::HirId)>,
 }
 
 impl<'a, 'tcx> InferBorrowKind<'a, 'tcx> {
@@ -1363,6 +1366,12 @@ impl<'a, 'tcx> InferBorrowKind<'a, 'tcx> {
 }
 
 impl<'a, 'tcx> euv::Delegate<'tcx> for InferBorrowKind<'a, 'tcx> {
+    fn fake_read(&mut self, place: Place<'tcx>, cause: FakeReadCause, diag_expr_id: hir::HirId) {
+        if let PlaceBase::Upvar(_) = place.base {
+            self.fake_reads.push((place, cause, diag_expr_id));
+        }
+    }
+
     fn consume(
         &mut self,
         place_with_id: &PlaceWithHirId<'tcx>,
@@ -1391,8 +1400,15 @@ impl<'a, 'tcx> euv::Delegate<'tcx> for InferBorrowKind<'a, 'tcx> {
             place_with_id, diag_expr_id, bk
         );
 
+        let place = restrict_repr_packed_field_ref_capture(
+            self.fcx.tcx,
+            self.fcx.param_env,
+            &place_with_id.place,
+        );
+        let place_with_id = PlaceWithHirId { place, ..*place_with_id };
+
         if !self.capture_information.contains_key(&place_with_id.place) {
-            self.init_capture_info_for_place(place_with_id, diag_expr_id);
+            self.init_capture_info_for_place(&place_with_id, diag_expr_id);
         }
 
         match bk {
@@ -1409,11 +1425,7 @@ impl<'a, 'tcx> euv::Delegate<'tcx> for InferBorrowKind<'a, 'tcx> {
     fn mutate(&mut self, assignee_place: &PlaceWithHirId<'tcx>, diag_expr_id: hir::HirId) {
         debug!("mutate(assignee_place={:?}, diag_expr_id={:?})", assignee_place, diag_expr_id);
 
-        if !self.capture_information.contains_key(&assignee_place.place) {
-            self.init_capture_info_for_place(assignee_place, diag_expr_id);
-        }
-
-        self.adjust_upvar_borrow_kind_for_mut(assignee_place, diag_expr_id);
+        self.borrow(assignee_place, diag_expr_id, ty::BorrowKind::MutBorrow);
     }
 }
 
@@ -1539,12 +1551,29 @@ fn should_do_migration_analysis(tcx: TyCtxt<'_>, closure_id: hir::HirId) -> bool
     !matches!(level, lint::Level::Allow)
 }
 
-fn migration_suggestion_for_2229(tcx: TyCtxt<'_>, need_migrations: &Vec<hir::HirId>) -> String {
-    let need_migrations_strings =
-        need_migrations.iter().map(|v| format!("{}", var_name(tcx, *v))).collect::<Vec<_>>();
-    let migrations_list_concat = need_migrations_strings.join(", ");
+/// Return a two string tuple (s1, s2)
+/// - s1: Line of code that is needed for the migration: eg: `let _ = (&x, ...)`.
+/// - s2: Comma separated names of the variables being migrated.
+fn migration_suggestion_for_2229(
+    tcx: TyCtxt<'_>,
+    need_migrations: &Vec<hir::HirId>,
+) -> (String, String) {
+    let need_migrations_variables =
+        need_migrations.iter().map(|v| var_name(tcx, *v)).collect::<Vec<_>>();
+
+    let migration_ref_concat =
+        need_migrations_variables.iter().map(|v| format!("&{}", v)).collect::<Vec<_>>().join(", ");
+
+    let migration_string = if 1 == need_migrations.len() {
+        format!("let _ = {}", migration_ref_concat)
+    } else {
+        format!("let _ = ({})", migration_ref_concat)
+    };
+
+    let migrated_variables_concat =
+        need_migrations_variables.iter().map(|v| format!("`{}`", v)).collect::<Vec<_>>().join(", ");
 
-    format!("drop(&({}));", migrations_list_concat)
+    (migration_string, migrated_variables_concat)
 }
 
 /// Helper function to determine if we need to escalate CaptureKind from
@@ -1559,7 +1588,7 @@ fn migration_suggestion_for_2229(tcx: TyCtxt<'_>, need_migrations: &Vec<hir::Hir
 /// If both the CaptureKind and Expression are considered to be equivalent,
 /// then `CaptureInfo` A is preferred. This can be useful in cases where we want to priortize
 /// expressions reported back to the user as part of diagnostics based on which appears earlier
-/// in the closure. This can be acheived simply by calling
+/// in the closure. This can be achieved simply by calling
 /// `determine_capture_info(existing_info, current_info)`. This works out because the
 /// expressions that occur earlier in the closure body than the current expression are processed before.
 /// Consider the following example
@@ -1657,7 +1686,7 @@ fn determine_place_ancestry_relation(
     let projections_b = &place_b.projections;
 
     let same_initial_projections =
-        projections_a.iter().zip(projections_b.iter()).all(|(proj_a, proj_b)| proj_a == proj_b);
+        iter::zip(projections_a, projections_b).all(|(proj_a, proj_b)| proj_a == proj_b);
 
     if same_initial_projections {
         // First min(n, m) projections are the same
diff --git a/compiler/rustc_typeck/src/check/wfcheck.rs b/compiler/rustc_typeck/src/check/wfcheck.rs
index 00c6550835b..887cc42a1dd 100644
--- a/compiler/rustc_typeck/src/check/wfcheck.rs
+++ b/compiler/rustc_typeck/src/check/wfcheck.rs
@@ -24,6 +24,7 @@ use rustc_trait_selection::opaque_types::may_define_opaque_type;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
 use rustc_trait_selection::traits::{self, ObligationCause, ObligationCauseCode};
 
+use std::iter;
 use std::ops::ControlFlow;
 
 /// Helper type of a temporary returned by `.for_item(...)`.
@@ -713,10 +714,11 @@ fn check_where_clauses<'tcx, 'fcx>(
     let generics = tcx.generics_of(def_id);
 
     let is_our_default = |def: &ty::GenericParamDef| match def.kind {
-        GenericParamDefKind::Type { has_default, .. } => {
+        GenericParamDefKind::Type { has_default, .. }
+        | GenericParamDefKind::Const { has_default } => {
             has_default && def.index >= generics.parent_count as u32
         }
-        _ => unreachable!(),
+        GenericParamDefKind::Lifetime => unreachable!(),
     };
 
     // Check that concrete defaults are well-formed. See test `type-check-defaults.rs`.
@@ -771,10 +773,15 @@ fn check_where_clauses<'tcx, 'fcx>(
 
                 fcx.tcx.mk_param_from_def(param)
             }
-
-            GenericParamDefKind::Const => {
-                // FIXME(const_generics_defaults)
-                fcx.tcx.mk_param_from_def(param)
+            GenericParamDefKind::Const { .. } => {
+                if is_our_default(param) {
+                    let default_ct = tcx.const_param_default(param.def_id);
+                    // Const params currently have to be concrete.
+                    assert!(!default_ct.needs_subst());
+                    default_ct.into()
+                } else {
+                    fcx.tcx.mk_param_from_def(param)
+                }
             }
         }
     });
@@ -857,7 +864,7 @@ fn check_where_clauses<'tcx, 'fcx>(
     debug!("check_where_clauses: predicates={:?}", predicates.predicates);
     assert_eq!(predicates.predicates.len(), predicates.spans.len());
     let wf_obligations =
-        predicates.predicates.iter().zip(predicates.spans.iter()).flat_map(|(&p, &sp)| {
+        iter::zip(&predicates.predicates, &predicates.spans).flat_map(|(&p, &sp)| {
             traits::wf::predicate_obligations(fcx, fcx.param_env, fcx.body_id, p, sp)
         });
 
@@ -879,8 +886,8 @@ fn check_fn_or_method<'fcx, 'tcx>(
     let sig = fcx.normalize_associated_types_in(span, sig);
     let sig = fcx.tcx.liberate_late_bound_regions(def_id, sig);
 
-    for (&input_ty, span) in sig.inputs().iter().zip(hir_decl.inputs.iter().map(|t| t.span)) {
-        fcx.register_wf_obligation(input_ty.into(), span, ObligationCauseCode::MiscObligation);
+    for (&input_ty, ty) in iter::zip(sig.inputs(), hir_decl.inputs) {
+        fcx.register_wf_obligation(input_ty.into(), ty.span, ObligationCauseCode::MiscObligation);
     }
     implied_bounds.extend(sig.inputs());
 
@@ -1060,13 +1067,14 @@ fn check_method_receiver<'fcx, 'tcx>(
     debug!("check_method_receiver: sig={:?}", sig);
 
     let self_ty = fcx.normalize_associated_types_in(span, self_ty);
-    let self_ty = fcx.tcx.liberate_late_bound_regions(method.def_id, ty::Binder::bind(self_ty));
+    let self_ty =
+        fcx.tcx.liberate_late_bound_regions(method.def_id, ty::Binder::bind(self_ty, fcx.tcx));
 
     let receiver_ty = sig.inputs()[0];
 
     let receiver_ty = fcx.normalize_associated_types_in(span, receiver_ty);
     let receiver_ty =
-        fcx.tcx.liberate_late_bound_regions(method.def_id, ty::Binder::bind(receiver_ty));
+        fcx.tcx.liberate_late_bound_regions(method.def_id, ty::Binder::bind(receiver_ty, fcx.tcx));
 
     if fcx.tcx.features().arbitrary_self_types {
         if !receiver_is_valid(fcx, span, receiver_ty, self_ty, true) {
diff --git a/compiler/rustc_typeck/src/check/writeback.rs b/compiler/rustc_typeck/src/check/writeback.rs
index af82a3bb4f5..e472add6e80 100644
--- a/compiler/rustc_typeck/src/check/writeback.rs
+++ b/compiler/rustc_typeck/src/check/writeback.rs
@@ -4,11 +4,15 @@
 
 use crate::check::FnCtxt;
 
+use rustc_data_structures::stable_map::FxHashMap;
 use rustc_errors::ErrorReported;
 use rustc_hir as hir;
+use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_infer::infer::error_reporting::TypeAnnotationNeeded::E0282;
 use rustc_infer::infer::InferCtxt;
+use rustc_middle::hir::place::Place as HirPlace;
+use rustc_middle::mir::FakeReadCause;
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, PointerCast};
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder};
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -56,7 +60,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
         wbcx.visit_body(body);
         wbcx.visit_min_capture_map();
-        wbcx.visit_upvar_capture_map();
+        wbcx.visit_fake_reads_map();
         wbcx.visit_closures();
         wbcx.visit_liberated_fn_sigs();
         wbcx.visit_fru_field_types();
@@ -74,9 +78,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         wbcx.typeck_results.treat_byte_string_as_slice =
             mem::take(&mut self.typeck_results.borrow_mut().treat_byte_string_as_slice);
 
-        wbcx.typeck_results.closure_captures =
-            mem::take(&mut self.typeck_results.borrow_mut().closure_captures);
-
         if self.is_tainted_by_errors() {
             // FIXME(eddyb) keep track of `ErrorReported` from where the error was emitted.
             wbcx.typeck_results.tainted_by_errors = Some(ErrorReported);
@@ -363,20 +364,25 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
         self.typeck_results.closure_min_captures = min_captures_wb;
     }
 
-    fn visit_upvar_capture_map(&mut self) {
-        for (upvar_id, upvar_capture) in self.fcx.typeck_results.borrow().upvar_capture_map.iter() {
-            let new_upvar_capture = match *upvar_capture {
-                ty::UpvarCapture::ByValue(span) => ty::UpvarCapture::ByValue(span),
-                ty::UpvarCapture::ByRef(ref upvar_borrow) => {
-                    ty::UpvarCapture::ByRef(ty::UpvarBorrow {
-                        kind: upvar_borrow.kind,
-                        region: self.tcx().lifetimes.re_erased,
-                    })
-                }
-            };
-            debug!("Upvar capture for {:?} resolved to {:?}", upvar_id, new_upvar_capture);
-            self.typeck_results.upvar_capture_map.insert(*upvar_id, new_upvar_capture);
+    fn visit_fake_reads_map(&mut self) {
+        let mut resolved_closure_fake_reads: FxHashMap<
+            DefId,
+            Vec<(HirPlace<'tcx>, FakeReadCause, hir::HirId)>,
+        > = Default::default();
+        for (closure_def_id, fake_reads) in
+            self.fcx.typeck_results.borrow().closure_fake_reads.iter()
+        {
+            let mut resolved_fake_reads = Vec::<(HirPlace<'tcx>, FakeReadCause, hir::HirId)>::new();
+            for (place, cause, hir_id) in fake_reads.iter() {
+                let locatable =
+                    self.tcx().hir().local_def_id_to_hir_id(closure_def_id.expect_local());
+
+                let resolved_fake_read = self.resolve(place.clone(), &locatable);
+                resolved_fake_reads.push((resolved_fake_read, *cause, *hir_id));
+            }
+            resolved_closure_fake_reads.insert(*closure_def_id, resolved_fake_reads);
         }
+        self.typeck_results.closure_fake_reads = resolved_closure_fake_reads;
     }
 
     fn visit_closures(&mut self) {
@@ -497,7 +503,8 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
             let mut skip_add = false;
 
             if let ty::Opaque(defin_ty_def_id, _substs) = *definition_ty.kind() {
-                if let hir::OpaqueTyOrigin::Misc = opaque_defn.origin {
+                if let hir::OpaqueTyOrigin::Misc | hir::OpaqueTyOrigin::TyAlias = opaque_defn.origin
+                {
                     if def_id == defin_ty_def_id {
                         debug!(
                             "skipping adding concrete definition for opaque type {:?} {:?}",
@@ -668,7 +675,7 @@ impl Locatable for hir::HirId {
 
 /// The Resolver. This is the type folding engine that detects
 /// unresolved types and so forth.
-crate struct Resolver<'cx, 'tcx> {
+struct Resolver<'cx, 'tcx> {
     tcx: TyCtxt<'tcx>,
     infcx: &'cx InferCtxt<'cx, 'tcx>,
     span: &'cx dyn Locatable,
@@ -679,7 +686,7 @@ crate struct Resolver<'cx, 'tcx> {
 }
 
 impl<'cx, 'tcx> Resolver<'cx, 'tcx> {
-    crate fn new(
+    fn new(
         fcx: &'cx FnCtxt<'cx, 'tcx>,
         span: &'cx dyn Locatable,
         body: &'tcx hir::Body<'tcx>,
diff --git a/compiler/rustc_typeck/src/coherence/builtin.rs b/compiler/rustc_typeck/src/coherence/builtin.rs
index 5b44cb7eae5..8cae61e8c22 100644
--- a/compiler/rustc_typeck/src/coherence/builtin.rs
+++ b/compiler/rustc_typeck/src/coherence/builtin.rs
@@ -246,7 +246,7 @@ fn visit_implementation_of_dispatch_from_dyn(tcx: TyCtxt<'_>, impl_did: LocalDef
                     ))
                     .emit();
                 } else {
-                    let mut fulfill_cx = TraitEngine::new(infcx.tcx);
+                    let mut fulfill_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
 
                     for field in coerced_fields {
                         let predicate = predicate_for_trait_def(
@@ -506,7 +506,7 @@ pub fn coerce_unsized_info(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUnsizedI
             }
         };
 
-        let mut fulfill_cx = TraitEngine::new(infcx.tcx);
+        let mut fulfill_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
 
         // Register an obligation for `A: Trait<B>`.
         let cause = traits::ObligationCause::misc(span, impl_hir_id);
diff --git a/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs b/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs
index 29654099992..c69389e7b43 100644
--- a/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs
+++ b/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs
@@ -24,8 +24,8 @@ impl InherentOverlapChecker<'tcx> {
     /// namespace.
     fn impls_have_common_items(
         &self,
-        impl_items1: &ty::AssociatedItems<'_>,
-        impl_items2: &ty::AssociatedItems<'_>,
+        impl_items1: &ty::AssocItems<'_>,
+        impl_items2: &ty::AssocItems<'_>,
     ) -> bool {
         let mut impl_items1 = &impl_items1;
         let mut impl_items2 = &impl_items2;
diff --git a/compiler/rustc_typeck/src/collect.rs b/compiler/rustc_typeck/src/collect.rs
index 3881d55ef91..1477418d5d8 100644
--- a/compiler/rustc_typeck/src/collect.rs
+++ b/compiler/rustc_typeck/src/collect.rs
@@ -15,6 +15,8 @@
 //! At present, however, we do run collection across all items in the
 //! crate as a kind of pass. This should eventually be factored away.
 
+// ignore-tidy-filelength
+
 use crate::astconv::{AstConv, SizedByDefault};
 use crate::bounds::Bounds;
 use crate::check::intrinsic::intrinsic_operation_unsafety;
@@ -43,13 +45,13 @@ use rustc_middle::ty::util::Discr;
 use rustc_middle::ty::util::IntTypeExt;
 use rustc_middle::ty::{self, AdtKind, Const, DefIdTree, ToPolyTraitRef, Ty, TyCtxt};
 use rustc_middle::ty::{ReprOptions, ToPredicate, WithConstness};
-use rustc_session::config::SanitizerSet;
 use rustc_session::lint;
 use rustc_session::parse::feature_err;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::spec::abi;
+use rustc_target::spec::{abi, SanitizerSet};
 use rustc_trait_selection::traits::error_reporting::suggestions::NextTypeParamName;
+use std::iter;
 
 mod item_bounds;
 mod type_of;
@@ -254,10 +256,15 @@ impl Visitor<'tcx> for CollectItemTypesVisitor<'tcx> {
                     self.tcx.ensure().type_of(def_id);
                 }
                 hir::GenericParamKind::Type { .. } => {}
-                hir::GenericParamKind::Const { .. } => {
+                hir::GenericParamKind::Const { default, .. } => {
                     let def_id = self.tcx.hir().local_def_id(param.hir_id);
                     self.tcx.ensure().type_of(def_id);
-                    // FIXME(const_generics_defaults)
+                    if let Some(default) = default {
+                        let default_def_id = self.tcx.hir().local_def_id(default.hir_id);
+                        // need to store default and type of default
+                        self.tcx.ensure().type_of(default_def_id);
+                        self.tcx.ensure().const_param_default(def_id);
+                    }
                 }
             }
         }
@@ -310,7 +317,7 @@ impl ItemCtxt<'tcx> {
     }
 
     pub fn to_ty(&self, ast_ty: &hir::Ty<'_>) -> Ty<'tcx> {
-        AstConv::ast_ty_to_ty(self, ast_ty)
+        <dyn AstConv<'_>>::ast_ty_to_ty(self, ast_ty)
     }
 
     pub fn hir_id(&self) -> hir::HirId {
@@ -729,8 +736,14 @@ fn convert_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
                 tcx.ensure().generics_of(item.def_id);
                 tcx.ensure().type_of(item.def_id);
                 tcx.ensure().predicates_of(item.def_id);
-                if let hir::ForeignItemKind::Fn(..) = item.kind {
-                    tcx.ensure().fn_sig(item.def_id);
+                match item.kind {
+                    hir::ForeignItemKind::Fn(..) => tcx.ensure().fn_sig(item.def_id),
+                    hir::ForeignItemKind::Static(..) => {
+                        let mut visitor = PlaceholderHirTyCollector::default();
+                        visitor.visit_foreign_item(item);
+                        placeholder_type_error(tcx, None, &[], visitor.0, false, None);
+                    }
+                    _ => (),
                 }
             }
         }
@@ -1096,7 +1109,7 @@ fn super_predicates_that_define_assoc_type(
         // Convert the bounds that follow the colon, e.g., `Bar + Zed` in `trait Foo: Bar + Zed`.
         let self_param_ty = tcx.types.self_param;
         let superbounds1 = if let Some(assoc_name) = assoc_name {
-            AstConv::compute_bounds_that_match_assoc_type(
+            <dyn AstConv<'_>>::compute_bounds_that_match_assoc_type(
                 &icx,
                 self_param_ty,
                 &bounds,
@@ -1105,7 +1118,13 @@ fn super_predicates_that_define_assoc_type(
                 assoc_name,
             )
         } else {
-            AstConv::compute_bounds(&icx, self_param_ty, &bounds, SizedByDefault::No, item.span)
+            <dyn AstConv<'_>>::compute_bounds(
+                &icx,
+                self_param_ty,
+                &bounds,
+                SizedByDefault::No,
+                item.span,
+            )
         };
 
         let superbounds1 = superbounds1.predicates(tcx, self_param_ty);
@@ -1232,7 +1251,8 @@ fn has_late_bound_regions<'tcx>(tcx: TyCtxt<'tcx>, node: Node<'tcx>) -> Option<S
             match self.tcx.named_region(lt.hir_id) {
                 Some(rl::Region::Static | rl::Region::EarlyBound(..)) => {}
                 Some(
-                    rl::Region::LateBound(debruijn, _, _) | rl::Region::LateBoundAnon(debruijn, _),
+                    rl::Region::LateBound(debruijn, _, _, _)
+                    | rl::Region::LateBoundAnon(debruijn, _, _),
                 ) if debruijn < self.outer_index => {}
                 Some(
                     rl::Region::LateBound(..)
@@ -1517,7 +1537,7 @@ fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
                         |lint| {
                             lint.build(
                                 "defaults for type parameters are only allowed in \
-                                 `struct`, `enum`, `type`, or `trait` definitions.",
+                                 `struct`, `enum`, `type`, or `trait` definitions",
                             )
                             .emit();
                         },
@@ -1543,13 +1563,21 @@ fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
             i += 1;
             Some(param_def)
         }
-        GenericParamKind::Const { .. } => {
+        GenericParamKind::Const { default, .. } => {
+            if !allow_defaults && default.is_some() {
+                tcx.sess.span_err(
+                    param.span,
+                    "defaults for const parameters are only allowed in \
+                    `struct`, `enum`, `type`, or `trait` definitions",
+                );
+            }
+
             let param_def = ty::GenericParamDef {
                 index: type_start + i as u32,
                 name: param.name.ident().name,
                 def_id: tcx.hir().local_def_id(param.hir_id).to_def_id(),
                 pure_wrt_drop: param.pure_wrt_drop,
-                kind: ty::GenericParamDefKind::Const,
+                kind: ty::GenericParamDefKind::Const { has_default: default.is_some() },
             };
             i += 1;
             Some(param_def)
@@ -1687,10 +1715,11 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::PolyFnSig<'_> {
                     }
                     diag.emit();
 
-                    ty::Binder::bind(fn_sig)
+                    ty::Binder::bind(fn_sig, tcx)
                 }
-                None => AstConv::ty_of_fn(
+                None => <dyn AstConv<'_>>::ty_of_fn(
                     &icx,
+                    hir_id,
                     sig.header.unsafety,
                     sig.header.abi,
                     &sig.decl,
@@ -1706,8 +1735,9 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::PolyFnSig<'_> {
             ident,
             generics,
             ..
-        }) => AstConv::ty_of_fn(
+        }) => <dyn AstConv<'_>>::ty_of_fn(
             &icx,
+            hir_id,
             header.unsafety,
             header.abi,
             decl,
@@ -1729,13 +1759,10 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::PolyFnSig<'_> {
             let ty = tcx.type_of(tcx.hir().get_parent_did(hir_id).to_def_id());
             let inputs =
                 data.fields().iter().map(|f| tcx.type_of(tcx.hir().local_def_id(f.hir_id)));
-            ty::Binder::bind(tcx.mk_fn_sig(
-                inputs,
-                ty,
-                false,
-                hir::Unsafety::Normal,
-                abi::Abi::Rust,
-            ))
+            ty::Binder::bind(
+                tcx.mk_fn_sig(inputs, ty, false, hir::Unsafety::Normal, abi::Abi::Rust),
+                tcx,
+            )
         }
 
         Expr(&hir::Expr { kind: hir::ExprKind::Closure(..), .. }) => {
@@ -1767,7 +1794,7 @@ fn impl_trait_ref(tcx: TyCtxt<'_>, def_id: DefId) -> Option<ty::TraitRef<'_>> {
     match tcx.hir().expect_item(hir_id).kind {
         hir::ItemKind::Impl(ref impl_) => impl_.of_trait.as_ref().map(|ast_trait_ref| {
             let selfty = tcx.type_of(def_id);
-            AstConv::instantiate_mono_trait_ref(&icx, ast_trait_ref, selfty)
+            <dyn AstConv<'_>>::instantiate_mono_trait_ref(&icx, ast_trait_ref, selfty)
         }),
         _ => bug!(),
     }
@@ -2018,8 +2045,8 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
             GenericParamKind::Lifetime { .. } => {
                 param.bounds.iter().for_each(|bound| match bound {
                     hir::GenericBound::Outlives(lt) => {
-                        let bound = AstConv::ast_region_to_region(&icx, &lt, None);
-                        let outlives = ty::Binder::bind(ty::OutlivesPredicate(region, bound));
+                        let bound = <dyn AstConv<'_>>::ast_region_to_region(&icx, &lt, None);
+                        let outlives = ty::Binder::bind(ty::OutlivesPredicate(region, bound), tcx);
                         predicates.insert((outlives.to_predicate(tcx), lt.span));
                     }
                     _ => bug!(),
@@ -2041,8 +2068,13 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
                 index += 1;
 
                 let sized = SizedByDefault::Yes;
-                let bounds =
-                    AstConv::compute_bounds(&icx, param_ty, &param.bounds, sized, param.span);
+                let bounds = <dyn AstConv<'_>>::compute_bounds(
+                    &icx,
+                    param_ty,
+                    &param.bounds,
+                    sized,
+                    param.span,
+                );
                 predicates.extend(bounds.predicates(tcx, param_ty));
             }
             GenericParamKind::Const { .. } => {
@@ -2059,6 +2091,7 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
         match predicate {
             hir::WherePredicate::BoundPredicate(bound_pred) => {
                 let ty = icx.to_ty(&bound_pred.bounded_ty);
+                let bound_vars = icx.tcx.late_bound_vars(bound_pred.bounded_ty.hir_id);
 
                 // Keep the type around in a dummy predicate, in case of no bounds.
                 // That way, `where Ty:` is not a complete noop (see #53696) and `Ty`
@@ -2074,9 +2107,13 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
                     } else {
                         let span = bound_pred.bounded_ty.span;
                         let re_root_empty = tcx.lifetimes.re_root_empty;
-                        let predicate = ty::Binder::bind(ty::PredicateKind::TypeOutlives(
-                            ty::OutlivesPredicate(ty, re_root_empty),
-                        ));
+                        let predicate = ty::Binder::bind_with_vars(
+                            ty::PredicateKind::TypeOutlives(ty::OutlivesPredicate(
+                                ty,
+                                re_root_empty,
+                            )),
+                            bound_vars,
+                        );
                         predicates.insert((predicate.to_predicate(tcx), span));
                     }
                 }
@@ -2091,19 +2128,21 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
                             };
 
                             let mut bounds = Bounds::default();
-                            let _ = AstConv::instantiate_poly_trait_ref(
+                            let _ = <dyn AstConv<'_>>::instantiate_poly_trait_ref(
                                 &icx,
-                                &poly_trait_ref,
+                                &poly_trait_ref.trait_ref,
+                                poly_trait_ref.span,
                                 constness,
                                 ty,
                                 &mut bounds,
+                                false,
                             );
                             predicates.extend(bounds.predicates(tcx, ty));
                         }
 
                         &hir::GenericBound::LangItemTrait(lang_item, span, hir_id, args) => {
                             let mut bounds = Bounds::default();
-                            AstConv::instantiate_lang_item_trait_ref(
+                            <dyn AstConv<'_>>::instantiate_lang_item_trait_ref(
                                 &icx,
                                 lang_item,
                                 span,
@@ -2116,11 +2155,15 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
                         }
 
                         hir::GenericBound::Outlives(lifetime) => {
-                            let region = AstConv::ast_region_to_region(&icx, lifetime, None);
+                            let region =
+                                <dyn AstConv<'_>>::ast_region_to_region(&icx, lifetime, None);
                             predicates.insert((
-                                ty::Binder::bind(ty::PredicateKind::TypeOutlives(
-                                    ty::OutlivesPredicate(ty, region),
-                                ))
+                                ty::Binder::bind_with_vars(
+                                    ty::PredicateKind::TypeOutlives(ty::OutlivesPredicate(
+                                        ty, region,
+                                    )),
+                                    bound_vars,
+                                )
                                 .to_predicate(tcx),
                                 lifetime.span,
                             ));
@@ -2130,11 +2173,11 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
             }
 
             hir::WherePredicate::RegionPredicate(region_pred) => {
-                let r1 = AstConv::ast_region_to_region(&icx, &region_pred.lifetime, None);
+                let r1 = <dyn AstConv<'_>>::ast_region_to_region(&icx, &region_pred.lifetime, None);
                 predicates.extend(region_pred.bounds.iter().map(|bound| {
                     let (r2, span) = match bound {
                         hir::GenericBound::Outlives(lt) => {
-                            (AstConv::ast_region_to_region(&icx, lt, None), lt.span)
+                            (<dyn AstConv<'_>>::ast_region_to_region(&icx, lt, None), lt.span)
                         }
                         _ => bug!(),
                     };
@@ -2200,10 +2243,11 @@ fn const_evaluatable_predicates_of<'tcx>(
         fn visit_anon_const(&mut self, c: &'tcx hir::AnonConst) {
             let def_id = self.tcx.hir().local_def_id(c.hir_id);
             let ct = ty::Const::from_anon_const(self.tcx, def_id);
-            if let ty::ConstKind::Unevaluated(def, substs, None) = ct.val {
+            if let ty::ConstKind::Unevaluated(uv) = ct.val {
+                assert_eq!(uv.promoted, None);
                 let span = self.tcx.hir().span(c.hir_id);
                 self.preds.insert((
-                    ty::PredicateKind::ConstEvaluatable(def, substs).to_predicate(self.tcx),
+                    ty::PredicateKind::ConstEvaluatable(uv.def, uv.substs).to_predicate(self.tcx),
                     span,
                 ));
             }
@@ -2341,7 +2385,14 @@ fn predicates_from_bound<'tcx>(
             };
 
             let mut bounds = Bounds::default();
-            let _ = astconv.instantiate_poly_trait_ref(tr, constness, param_ty, &mut bounds);
+            let _ = astconv.instantiate_poly_trait_ref(
+                &tr.trait_ref,
+                tr.span,
+                constness,
+                param_ty,
+                &mut bounds,
+                false,
+            );
             bounds.predicates(astconv.tcx(), param_ty)
         }
         hir::GenericBound::LangItemTrait(lang_item, span, hir_id, args) => {
@@ -2377,8 +2428,10 @@ fn compute_sig_of_foreign_fn_decl<'tcx>(
     } else {
         hir::Unsafety::Unsafe
     };
-    let fty = AstConv::ty_of_fn(
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+    let fty = <dyn AstConv<'_>>::ty_of_fn(
         &ItemCtxt::new(tcx, def_id),
+        hir_id,
         unsafety,
         abi,
         decl,
@@ -2413,7 +2466,7 @@ fn compute_sig_of_foreign_fn_decl<'tcx>(
                     .emit();
             }
         };
-        for (input, ty) in decl.inputs.iter().zip(fty.inputs().skip_binder()) {
+        for (input, ty) in iter::zip(decl.inputs, fty.inputs().skip_binder()) {
             check(&input, ty)
         }
         if let hir::FnRetTy::Return(ref ty) = decl.output {
@@ -2666,7 +2719,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, id: DefId) -> CodegenFnAttrs {
         } else if tcx.sess.check_name(attr, sym::used) {
             codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED;
         } else if tcx.sess.check_name(attr, sym::cmse_nonsecure_entry) {
-            if tcx.fn_sig(id).abi() != abi::Abi::C {
+            if !matches!(tcx.fn_sig(id).abi(), abi::Abi::C { .. }) {
                 struct_span_err!(
                     tcx.sess,
                     attr.span,
@@ -2838,6 +2891,36 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, id: DefId) -> CodegenFnAttrs {
                     None
                 }
             };
+        } else if tcx.sess.check_name(attr, sym::repr) {
+            codegen_fn_attrs.alignment = match attr.meta_item_list() {
+                Some(items) => match items.as_slice() {
+                    [item] => match item.name_value_literal() {
+                        Some((sym::align, literal)) => {
+                            let alignment = rustc_attr::parse_alignment(&literal.kind);
+
+                            match alignment {
+                                Ok(align) => Some(align),
+                                Err(msg) => {
+                                    struct_span_err!(
+                                        tcx.sess.diagnostic(),
+                                        attr.span,
+                                        E0589,
+                                        "invalid `repr(align)` attribute: {}",
+                                        msg
+                                    )
+                                    .emit();
+
+                                    None
+                                }
+                            }
+                        }
+                        _ => None,
+                    },
+                    [] => None,
+                    _ => None,
+                },
+                None => None,
+            };
         }
     }
 
diff --git a/compiler/rustc_typeck/src/collect/item_bounds.rs b/compiler/rustc_typeck/src/collect/item_bounds.rs
index fe18dc5ed0c..a5b36445aae 100644
--- a/compiler/rustc_typeck/src/collect/item_bounds.rs
+++ b/compiler/rustc_typeck/src/collect/item_bounds.rs
@@ -25,7 +25,7 @@ fn associated_type_bounds<'tcx>(
         InternalSubsts::identity_for_item(tcx, assoc_item_def_id),
     );
 
-    let bounds = AstConv::compute_bounds(
+    let bounds = <dyn AstConv<'_>>::compute_bounds(
         &ItemCtxt::new(tcx, assoc_item_def_id),
         item_ty,
         &bounds,
@@ -66,7 +66,7 @@ fn opaque_type_bounds<'tcx>(
         let item_ty =
             tcx.mk_opaque(opaque_def_id, InternalSubsts::identity_for_item(tcx, opaque_def_id));
 
-        let bounds = AstConv::compute_bounds(
+        let bounds = <dyn AstConv<'_>>::compute_bounds(
             &ItemCtxt::new(tcx, opaque_def_id),
             item_ty,
             &bounds,
diff --git a/compiler/rustc_typeck/src/collect/type_of.rs b/compiler/rustc_typeck/src/collect/type_of.rs
index e6f771cfd53..d8eea1ad80b 100644
--- a/compiler/rustc_typeck/src/collect/type_of.rs
+++ b/compiler/rustc_typeck/src/collect/type_of.rs
@@ -83,7 +83,7 @@ pub(super) fn opt_const_param_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<
                     return generics
                         .params
                         .iter()
-                        .filter(|param| matches!(param.kind, ty::GenericParamDefKind::Const))
+                        .filter(|param| matches!(param.kind, ty::GenericParamDefKind::Const { .. }))
                         .nth(arg_index)
                         .map(|param| param.def_id);
                 }
@@ -121,7 +121,7 @@ pub(super) fn opt_const_param_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<
                 tcx.generics_of(type_dependent_def)
                     .params
                     .iter()
-                    .filter(|param| matches!(param.kind, ty::GenericParamDefKind::Const))
+                    .filter(|param| matches!(param.kind, ty::GenericParamDefKind::Const { .. }))
                     .nth(idx)
                     .map(|param| param.def_id)
             }
@@ -211,7 +211,7 @@ pub(super) fn opt_const_param_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<
                 generics
                     .params
                     .iter()
-                    .filter(|param| matches!(param.kind, ty::GenericParamDefKind::Const))
+                    .filter(|param| matches!(param.kind, ty::GenericParamDefKind::Const { .. }))
                     .nth(arg_index)
                     .map(|param| param.def_id)
             }
@@ -430,12 +430,27 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
                     tcx.typeck(def_id).node_type(anon_const.hir_id)
                 }
 
+                Node::Expr(&Expr { kind: ExprKind::InlineAsm(ia), .. })
+                    if ia.operands.iter().any(|(op, _op_sp)| match op {
+                        hir::InlineAsmOperand::Const { anon_const } => anon_const.hir_id == hir_id,
+                        _ => false,
+                    }) =>
+                {
+                    tcx.typeck(def_id).node_type(hir_id)
+                }
+
                 Node::Variant(Variant { disr_expr: Some(ref e), .. }) if e.hir_id == hir_id => tcx
                     .adt_def(tcx.hir().get_parent_did(hir_id).to_def_id())
                     .repr
                     .discr_type()
                     .to_ty(tcx),
 
+                Node::GenericParam(&GenericParam {
+                    hir_id: param_hir_id,
+                    kind: GenericParamKind::Const { default: Some(ct), .. },
+                    ..
+                }) if ct.hir_id == hir_id => tcx.type_of(tcx.hir().local_def_id(param_hir_id)),
+
                 x => tcx.ty_error_with_message(
                     DUMMY_SP,
                     &format!("unexpected const parent in type_of_def_id(): {:?}", x),
@@ -722,11 +737,12 @@ fn infer_placeholder_type(
                 format!("{}: {}", item_ident, ty),
                 Applicability::MachineApplicable,
             )
-            .emit();
+            .emit_unless(ty.references_error());
         }
         None => {
             let mut diag = bad_placeholder_type(tcx, vec![span]);
-            if !matches!(ty.kind(), ty::Error(_)) {
+
+            if !ty.references_error() {
                 diag.span_suggestion(
                     span,
                     "replace `_` with the correct type",
@@ -734,6 +750,7 @@ fn infer_placeholder_type(
                     Applicability::MaybeIncorrect,
                 );
             }
+
             diag.emit();
         }
     }
diff --git a/compiler/rustc_typeck/src/expr_use_visitor.rs b/compiler/rustc_typeck/src/expr_use_visitor.rs
index 52110af4792..532ee00daf8 100644
--- a/compiler/rustc_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_typeck/src/expr_use_visitor.rs
@@ -5,8 +5,9 @@
 pub use self::ConsumeMode::*;
 
 // Export these here so that Clippy can use them.
-pub use rustc_middle::hir::place::{PlaceBase, PlaceWithHirId, Projection};
+pub use rustc_middle::hir::place::{Place, PlaceBase, PlaceWithHirId, Projection};
 
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir as hir;
 use rustc_hir::def::Res;
 use rustc_hir::def_id::LocalDefId;
@@ -14,8 +15,10 @@ use rustc_hir::PatKind;
 use rustc_index::vec::Idx;
 use rustc_infer::infer::InferCtxt;
 use rustc_middle::hir::place::ProjectionKind;
+use rustc_middle::mir::FakeReadCause;
 use rustc_middle::ty::{self, adjustment, TyCtxt};
 use rustc_target::abi::VariantIdx;
+use std::iter;
 
 use crate::mem_categorization as mc;
 
@@ -51,6 +54,9 @@ pub trait Delegate<'tcx> {
     // The path at `assignee_place` is being assigned to.
     // `diag_expr_id` is the id used for diagnostics (see `consume` for more details).
     fn mutate(&mut self, assignee_place: &PlaceWithHirId<'tcx>, diag_expr_id: hir::HirId);
+
+    // The `place` should be a fake read because of specified `cause`.
+    fn fake_read(&mut self, place: Place<'tcx>, cause: FakeReadCause, diag_expr_id: hir::HirId);
 }
 
 #[derive(Copy, Clone, PartialEq, Debug)]
@@ -229,7 +235,66 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
 
             hir::ExprKind::Match(ref discr, arms, _) => {
                 let discr_place = return_if_err!(self.mc.cat_expr(&discr));
-                self.borrow_expr(&discr, ty::ImmBorrow);
+
+                // Matching should not always be considered a use of the place, hence
+                // discr does not necessarily need to be borrowed.
+                // We only want to borrow discr if the pattern contain something other
+                // than wildcards.
+                let ExprUseVisitor { ref mc, body_owner: _, delegate: _ } = *self;
+                let mut needs_to_be_read = false;
+                for arm in arms.iter() {
+                    return_if_err!(mc.cat_pattern(discr_place.clone(), &arm.pat, |place, pat| {
+                        match &pat.kind {
+                            PatKind::Binding(.., opt_sub_pat) => {
+                                // If the opt_sub_pat is None, than the binding does not count as
+                                // a wildcard for the purpose of borrowing discr.
+                                if opt_sub_pat.is_none() {
+                                    needs_to_be_read = true;
+                                }
+                            }
+                            PatKind::TupleStruct(..)
+                            | PatKind::Path(..)
+                            | PatKind::Struct(..)
+                            | PatKind::Tuple(..) => {
+                                // If the PatKind is a TupleStruct, Struct or Tuple then we want to check
+                                // whether the Variant is a MultiVariant or a SingleVariant. We only want
+                                // to borrow discr if it is a MultiVariant.
+                                // If it is a SingleVariant and creates a binding we will handle that when
+                                // this callback gets called again.
+                                if let ty::Adt(def, _) = place.place.base_ty.kind() {
+                                    if def.variants.len() > 1 {
+                                        needs_to_be_read = true;
+                                    }
+                                }
+                            }
+                            PatKind::Lit(_) => {
+                                // If the PatKind is a Lit then we want
+                                // to borrow discr.
+                                needs_to_be_read = true;
+                            }
+                            _ => {}
+                        }
+                    }));
+                }
+
+                if needs_to_be_read {
+                    self.borrow_expr(&discr, ty::ImmBorrow);
+                } else {
+                    let closure_def_id = match discr_place.place.base {
+                        PlaceBase::Upvar(upvar_id) => Some(upvar_id.closure_expr_id.to_def_id()),
+                        _ => None,
+                    };
+
+                    self.delegate.fake_read(
+                        discr_place.place.clone(),
+                        FakeReadCause::ForMatchedPlace(closure_def_id),
+                        discr_place.hir_id,
+                    );
+
+                    // We always want to walk the discriminant. We want to make sure, for instance,
+                    // that the discriminant has been initialized.
+                    self.walk_expr(&discr);
+                }
 
                 // treatment of the discriminant is handled while walking the arms.
                 for arm in arms {
@@ -253,7 +318,6 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                 for (op, _op_sp) in asm.operands {
                     match op {
                         hir::InlineAsmOperand::In { expr, .. }
-                        | hir::InlineAsmOperand::Const { expr, .. }
                         | hir::InlineAsmOperand::Sym { expr, .. } => self.consume_expr(expr),
                         hir::InlineAsmOperand::Out { expr, .. } => {
                             if let Some(expr) = expr {
@@ -269,12 +333,13 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                                 self.mutate_expr(out_expr);
                             }
                         }
+                        hir::InlineAsmOperand::Const { .. } => {}
                     }
                 }
             }
 
             hir::ExprKind::LlvmInlineAsm(ref ia) => {
-                for (o, output) in ia.inner.outputs.iter().zip(ia.outputs_exprs) {
+                for (o, output) in iter::zip(&ia.inner.outputs, ia.outputs_exprs) {
                     if o.is_indirect {
                         self.consume_expr(output);
                     } else {
@@ -397,7 +462,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
 
     fn walk_struct_expr(
         &mut self,
-        fields: &[hir::Field<'_>],
+        fields: &[hir::ExprField<'_>],
         opt_with: &Option<&'hir hir::Expr<'_>>,
     ) {
         // Consume the expressions supplying values for each field.
@@ -518,6 +583,16 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
     }
 
     fn walk_arm(&mut self, discr_place: &PlaceWithHirId<'tcx>, arm: &hir::Arm<'_>) {
+        let closure_def_id = match discr_place.place.base {
+            PlaceBase::Upvar(upvar_id) => Some(upvar_id.closure_expr_id.to_def_id()),
+            _ => None,
+        };
+
+        self.delegate.fake_read(
+            discr_place.place.clone(),
+            FakeReadCause::ForMatchedPlace(closure_def_id),
+            discr_place.hir_id,
+        );
         self.walk_pat(discr_place, &arm.pat);
 
         if let Some(hir::Guard::If(ref e)) = arm.guard {
@@ -530,6 +605,16 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
     /// Walks a pat that occurs in isolation (i.e., top-level of fn argument or
     /// let binding, and *not* a match arm or nested pat.)
     fn walk_irrefutable_pat(&mut self, discr_place: &PlaceWithHirId<'tcx>, pat: &hir::Pat<'_>) {
+        let closure_def_id = match discr_place.place.base {
+            PlaceBase::Upvar(upvar_id) => Some(upvar_id.closure_expr_id.to_def_id()),
+            _ => None,
+        };
+
+        self.delegate.fake_read(
+            discr_place.place.clone(),
+            FakeReadCause::ForLet(closure_def_id),
+            discr_place.hir_id,
+        );
         self.walk_pat(discr_place, pat);
     }
 
@@ -586,7 +671,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
     /// In the following example the closures `c` only captures `p.x`` even though `incr`
     /// is a capture of the nested closure
     ///
-    /// ```rust,ignore(cannot-test-this-because-pseduo-code)
+    /// ```rust,ignore(cannot-test-this-because-pseudo-code)
     /// let p = ..;
     /// let c = || {
     ///    let incr = 10;
@@ -597,6 +682,14 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
     /// - When reporting the Place back to the Delegate, ensure that the UpvarId uses the enclosing
     /// closure as the DefId.
     fn walk_captures(&mut self, closure_expr: &hir::Expr<'_>) {
+        fn upvar_is_local_variable(
+            upvars: Option<&'tcx FxIndexMap<hir::HirId, hir::Upvar>>,
+            upvar_id: &hir::HirId,
+            body_owner_is_closure: bool,
+        ) -> bool {
+            upvars.map(|upvars| !upvars.contains_key(upvar_id)).unwrap_or(body_owner_is_closure)
+        }
+
         debug!("walk_captures({:?})", closure_expr);
 
         let closure_def_id = self.tcx().hir().local_def_id(closure_expr.hir_id).to_def_id();
@@ -608,6 +701,46 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
             ty::Closure(..) | ty::Generator(..)
         );
 
+        // If we have a nested closure, we want to include the fake reads present in the nested closure.
+        if let Some(fake_reads) = self.mc.typeck_results.closure_fake_reads.get(&closure_def_id) {
+            for (fake_read, cause, hir_id) in fake_reads.iter() {
+                match fake_read.base {
+                    PlaceBase::Upvar(upvar_id) => {
+                        if upvar_is_local_variable(
+                            upvars,
+                            &upvar_id.var_path.hir_id,
+                            body_owner_is_closure,
+                        ) {
+                            // The nested closure might be fake reading the current (enclosing) closure's local variables.
+                            // The only places we want to fake read before creating the parent closure are the ones that
+                            // are not local to it/ defined by it.
+                            //
+                            // ```rust,ignore(cannot-test-this-because-pseudo-code)
+                            // let v1 = (0, 1);
+                            // let c = || { // fake reads: v1
+                            //    let v2 = (0, 1);
+                            //    let e = || { // fake reads: v1, v2
+                            //       let (_, t1) = v1;
+                            //       let (_, t2) = v2;
+                            //    }
+                            // }
+                            // ```
+                            // This check is performed when visiting the body of the outermost closure (`c`) and ensures
+                            // that we don't add a fake read of v2 in c.
+                            continue;
+                        }
+                    }
+                    _ => {
+                        bug!(
+                            "Do not know how to get HirId out of Rvalue and StaticItem {:?}",
+                            fake_read.base
+                        );
+                    }
+                };
+                self.delegate.fake_read(fake_read.clone(), *cause, *hir_id);
+            }
+        }
+
         if let Some(min_captures) = self.mc.typeck_results.closure_min_captures.get(&closure_def_id)
         {
             for (var_hir_id, min_list) in min_captures.iter() {
diff --git a/compiler/rustc_typeck/src/impl_wf_check.rs b/compiler/rustc_typeck/src/impl_wf_check.rs
index 7713381e62e..12409468605 100644
--- a/compiler/rustc_typeck/src/impl_wf_check.rs
+++ b/compiler/rustc_typeck/src/impl_wf_check.rs
@@ -173,7 +173,7 @@ fn enforce_impl_params_are_constrained(
                     );
                 }
             }
-            ty::GenericParamDefKind::Const => {
+            ty::GenericParamDefKind::Const { .. } => {
                 let param_ct = ty::ParamConst::for_def(param);
                 if !input_parameters.contains(&cgp::Parameter::from(param_ct)) {
                     report_unused_parameter(
diff --git a/compiler/rustc_typeck/src/lib.rs b/compiler/rustc_typeck/src/lib.rs
index 6ddc26efeae..190744fe6f1 100644
--- a/compiler/rustc_typeck/src/lib.rs
+++ b/compiler/rustc_typeck/src/lib.rs
@@ -63,8 +63,9 @@ This API is completely unstable and subject to change.
 #![feature(format_args_capture)]
 #![feature(in_band_lifetimes)]
 #![feature(is_sorted)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(try_blocks)]
 #![feature(never_type)]
 #![feature(slice_partition_dedup)]
@@ -118,14 +119,19 @@ use astconv::AstConv;
 use bounds::Bounds;
 
 fn require_c_abi_if_c_variadic(tcx: TyCtxt<'_>, decl: &hir::FnDecl<'_>, abi: Abi, span: Span) {
-    if decl.c_variadic && !(abi == Abi::C || abi == Abi::Cdecl) {
-        let mut err = struct_span_err!(
-            tcx.sess,
-            span,
-            E0045,
-            "C-variadic function must have C or cdecl calling convention"
-        );
-        err.span_label(span, "C-variadics require C or cdecl calling convention").emit();
+    match (decl.c_variadic, abi) {
+        // The function has the correct calling convention, or isn't a "C-variadic" function.
+        (false, _) | (true, Abi::C { .. }) | (true, Abi::Cdecl) => {}
+        // The function is a "C-variadic" function with an incorrect calling convention.
+        (true, _) => {
+            let mut err = struct_span_err!(
+                tcx.sess,
+                span,
+                E0045,
+                "C-variadic function must have C or cdecl calling convention"
+            );
+            err.span_label(span, "C-variadics require C or cdecl calling convention").emit();
+        }
     }
 }
 
@@ -137,7 +143,7 @@ fn require_same_types<'tcx>(
 ) -> bool {
     tcx.infer_ctxt().enter(|ref infcx| {
         let param_env = ty::ParamEnv::empty();
-        let mut fulfill_cx = TraitEngine::new(infcx.tcx);
+        let mut fulfill_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
         match infcx.at(&cause, param_env).eq(expected, actual) {
             Ok(InferOk { obligations, .. }) => {
                 fulfill_cx.register_predicate_obligations(infcx, obligations);
@@ -201,7 +207,8 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: LocalDefId) {
                         error = true;
                     }
 
-                    for attr in it.attrs {
+                    let attrs = tcx.hir().attrs(main_id);
+                    for attr in attrs {
                         if tcx.sess.check_name(attr, sym::track_caller) {
                             tcx.sess
                                 .struct_span_err(
@@ -300,7 +307,8 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: LocalDefId) {
                         error = true;
                     }
 
-                    for attr in it.attrs {
+                    let attrs = tcx.hir().attrs(start_id);
+                    for attr in attrs {
                         if tcx.sess.check_name(attr, sym::track_caller) {
                             tcx.sess
                                 .struct_span_err(
@@ -422,7 +430,7 @@ pub fn hir_ty_to_ty<'tcx>(tcx: TyCtxt<'tcx>, hir_ty: &hir::Ty<'_>) -> Ty<'tcx> {
     let env_node_id = tcx.hir().get_parent_item(hir_ty.hir_id);
     let env_def_id = tcx.hir().local_def_id(env_node_id);
     let item_cx = self::collect::ItemCtxt::new(tcx, env_def_id.to_def_id());
-    item_cx.to_ty(hir_ty)
+    <dyn AstConv<'_>>::ast_ty_to_ty(&item_cx, hir_ty)
 }
 
 pub fn hir_trait_to_predicates<'tcx>(
@@ -437,7 +445,7 @@ pub fn hir_trait_to_predicates<'tcx>(
     let env_def_id = tcx.hir().local_def_id(env_hir_id);
     let item_cx = self::collect::ItemCtxt::new(tcx, env_def_id.to_def_id());
     let mut bounds = Bounds::default();
-    let _ = AstConv::instantiate_poly_trait_ref_inner(
+    let _ = <dyn AstConv<'_>>::instantiate_poly_trait_ref(
         &item_cx,
         hir_trait,
         DUMMY_SP,
diff --git a/compiler/rustc_typeck/src/variance/solve.rs b/compiler/rustc_typeck/src/variance/solve.rs
index 2d3369cba7a..1a4d88ced0e 100644
--- a/compiler/rustc_typeck/src/variance/solve.rs
+++ b/compiler/rustc_typeck/src/variance/solve.rs
@@ -78,7 +78,7 @@ impl<'a, 'tcx> SolveContext<'a, 'tcx> {
 
         // Make all const parameters invariant.
         for param in generics.params.iter() {
-            if let ty::GenericParamDefKind::Const = param.kind {
+            if let ty::GenericParamDefKind::Const { .. } = param.kind {
                 variances[param.index as usize] = ty::Invariant;
             }
         }