Split TypeFolder and FallibleTypeFolder

10 files changed, 395 insertions, 186 deletions

diff --git a/compiler/rustc_macros/src/type_foldable.rs b/compiler/rustc_macros/src/type_foldable.rs
index c4f584dca43..85051311bee 100644
--- a/compiler/rustc_macros/src/type_foldable.rs
+++ b/compiler/rustc_macros/src/type_foldable.rs
@@ -14,27 +14,13 @@ pub(super) fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_m
 
     s.add_bounds(synstructure::AddBounds::Generics);
     s.bind_with(|_| synstructure::BindStyle::Move);
-    let body_fold = s.each_variant(|vi| {
+    let try_body_fold = s.each_variant(|vi| {
         let bindings = vi.bindings();
         vi.construct(|_, index| {
             let bind = &bindings[index];
 
-            let mut fixed = false;
-
             // retain value of fields with #[type_foldable(identity)]
-            bind.ast().attrs.iter().for_each(|x| {
-                if !x.path().is_ident("type_foldable") {
-                    return;
-                }
-                let _ = x.parse_nested_meta(|nested| {
-                    if nested.path.is_ident("identity") {
-                        fixed = true;
-                    }
-                    Ok(())
-                });
-            });
-
-            if fixed {
+            if has_ignore_attr(&bind.ast().attrs, "type_foldable", "identity") {
                 bind.to_token_stream()
             } else {
                 quote! {
@@ -44,6 +30,22 @@ pub(super) fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_m
         })
     });
 
+    let body_fold = s.each_variant(|vi| {
+        let bindings = vi.bindings();
+        vi.construct(|_, index| {
+            let bind = &bindings[index];
+
+            // retain value of fields with #[type_foldable(identity)]
+            if has_ignore_attr(&bind.ast().attrs, "type_foldable", "identity") {
+                bind.to_token_stream()
+            } else {
+                quote! {
+                    ::rustc_middle::ty::TypeFoldable::fold_with(#bind, __folder)
+                }
+            }
+        })
+    });
+
     s.bound_impl(
         quote!(::rustc_middle::ty::TypeFoldable<::rustc_middle::ty::TyCtxt<'tcx>>),
         quote! {
@@ -51,8 +53,32 @@ pub(super) fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_m
                 self,
                 __folder: &mut __F
             ) -> Result<Self, __F::Error> {
-                Ok(match self { #body_fold })
+                Ok(match self { #try_body_fold })
+            }
+
+            fn fold_with<__F: ::rustc_middle::ty::TypeFolder<::rustc_middle::ty::TyCtxt<'tcx>>>(
+                self,
+                __folder: &mut __F
+            ) -> Self {
+                match self { #body_fold }
             }
         },
     )
 }
+
+fn has_ignore_attr(attrs: &[syn::Attribute], name: &'static str, meta: &'static str) -> bool {
+    let mut ignored = false;
+    attrs.iter().for_each(|attr| {
+        if !attr.path().is_ident(name) {
+            return;
+        }
+        let _ = attr.parse_nested_meta(|nested| {
+            if nested.path.is_ident(meta) {
+                ignored = true;
+            }
+            Ok(())
+        });
+    });
+
+    ignored
+}
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index 3cd148cd442..5b860374496 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -39,15 +39,6 @@ pub type CanonicalVarInfo<'tcx> = ir::CanonicalVarInfo<TyCtxt<'tcx>>;
 pub type CanonicalVarValues<'tcx> = ir::CanonicalVarValues<TyCtxt<'tcx>>;
 pub type CanonicalVarInfos<'tcx> = &'tcx List<CanonicalVarInfo<'tcx>>;
 
-impl<'tcx> ty::TypeFoldable<TyCtxt<'tcx>> for CanonicalVarInfos<'tcx> {
-    fn try_fold_with<F: ty::FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_canonical_var_infos(v))
-    }
-}
-
 /// When we canonicalize a value to form a query, we wind up replacing
 /// various parts of it with canonical variables. This struct stores
 /// those replaced bits to remember for when we process the query
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index ff9d32ebb71..c7561f8afef 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -931,6 +931,8 @@ pub enum TerminatorKind<'tcx> {
         asm_macro: InlineAsmMacro,
 
         /// The template for the inline assembly, with placeholders.
+        #[type_foldable(identity)]
+        #[type_visitable(ignore)]
         template: &'tcx [InlineAsmTemplatePiece],
 
         /// The operands for the inline assembly, as `Operand`s or `Place`s.
@@ -941,6 +943,8 @@ pub enum TerminatorKind<'tcx> {
 
         /// Source spans for each line of the inline assembly code. These are
         /// used to map assembler errors back to the line in the source code.
+        #[type_foldable(identity)]
+        #[type_visitable(ignore)]
         line_spans: &'tcx [Span],
 
         /// Valid targets for the inline assembly.
diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs
index 1f04937232d..8de64b3bfac 100644
--- a/compiler/rustc_middle/src/ty/generic_args.rs
+++ b/compiler/rustc_middle/src/ty/generic_args.rs
@@ -17,7 +17,7 @@ use smallvec::SmallVec;
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::{
     self, ClosureArgs, CoroutineArgs, CoroutineClosureArgs, FallibleTypeFolder, InlineConstArgs,
-    Lift, List, Ty, TyCtxt, TypeFoldable, TypeVisitable, TypeVisitor, VisitorResult,
+    Lift, List, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeVisitable, TypeVisitor, VisitorResult,
     walk_visitable_list,
 };
 
@@ -337,6 +337,14 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for GenericArg<'tcx> {
             GenericArgKind::Const(ct) => ct.try_fold_with(folder).map(Into::into),
         }
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        match self.unpack() {
+            GenericArgKind::Lifetime(lt) => lt.fold_with(folder).into(),
+            GenericArgKind::Type(ty) => ty.fold_with(folder).into(),
+            GenericArgKind::Const(ct) => ct.fold_with(folder).into(),
+        }
+    }
 }
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for GenericArg<'tcx> {
@@ -606,6 +614,27 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for GenericArgsRef<'tcx> {
                 }
             }
             0 => Ok(self),
+            _ => ty::util::try_fold_list(self, folder, |tcx, v| tcx.mk_args(v)),
+        }
+    }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        // See justification for this behavior in `try_fold_with`.
+        match self.len() {
+            1 => {
+                let param0 = self[0].fold_with(folder);
+                if param0 == self[0] { self } else { folder.cx().mk_args(&[param0]) }
+            }
+            2 => {
+                let param0 = self[0].fold_with(folder);
+                let param1 = self[1].fold_with(folder);
+                if param0 == self[0] && param1 == self[1] {
+                    self
+                } else {
+                    folder.cx().mk_args(&[param0, param1])
+                }
+            }
+            0 => self,
             _ => ty::util::fold_list(self, folder, |tcx, v| tcx.mk_args(v)),
         }
     }
@@ -641,6 +670,22 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<Ty<'tcx>> {
                     Ok(folder.cx().mk_type_list(&[param0, param1]))
                 }
             }
+            _ => ty::util::try_fold_list(self, folder, |tcx, v| tcx.mk_type_list(v)),
+        }
+    }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        // See comment justifying behavior in `try_fold_with`.
+        match self.len() {
+            2 => {
+                let param0 = self[0].fold_with(folder);
+                let param1 = self[1].fold_with(folder);
+                if param0 == self[0] && param1 == self[1] {
+                    self
+                } else {
+                    folder.cx().mk_type_list(&[param0, param1])
+                }
+            }
             _ => ty::util::fold_list(self, folder, |tcx, v| tcx.mk_type_list(v)),
         }
     }
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 61e869f5de4..673e1cafa9d 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -537,6 +537,13 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Term<'tcx> {
             ty::TermKind::Const(ct) => ct.try_fold_with(folder).map(Into::into),
         }
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        match self.unpack() {
+            ty::TermKind::Ty(ty) => ty.fold_with(folder).into(),
+            ty::TermKind::Const(ct) => ct.fold_with(folder).into(),
+        }
+    }
 }
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for Term<'tcx> {
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 798ef352c40..40eef541423 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -6,20 +6,19 @@
 use std::fmt::{self, Debug};
 
 use rustc_abi::TyAndLayout;
-use rustc_ast::InlineAsmTemplatePiece;
 use rustc_hir::def::Namespace;
 use rustc_hir::def_id::LocalDefId;
-use rustc_span::Span;
 use rustc_span::source_map::Spanned;
-use rustc_type_ir::{ConstKind, VisitorResult, try_visit};
+use rustc_type_ir::{ConstKind, TypeFolder, VisitorResult, try_visit};
 
 use super::print::PrettyPrinter;
 use super::{GenericArg, GenericArgKind, Pattern, Region};
+use crate::infer::canonical::CanonicalVarInfos;
 use crate::mir::PlaceElem;
 use crate::ty::print::{FmtPrinter, Printer, with_no_trimmed_paths};
 use crate::ty::{
-    self, FallibleTypeFolder, InferConst, Lift, Term, TermKind, Ty, TyCtxt, TypeFoldable,
-    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitor,
+    self, FallibleTypeFolder, Lift, Term, TermKind, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable,
+    TypeSuperVisitable, TypeVisitable, TypeVisitor,
 };
 
 impl fmt::Debug for ty::TraitDef {
@@ -271,6 +270,7 @@ TrivialTypeTraversalImpls! {
     crate::ty::AssocKind,
     crate::ty::BoundRegion,
     crate::ty::BoundVar,
+    crate::ty::InferConst,
     crate::ty::Placeholder<crate::ty::BoundRegion>,
     crate::ty::Placeholder<crate::ty::BoundTy>,
     crate::ty::Placeholder<ty::BoundVar>,
@@ -337,24 +337,6 @@ impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::AdtDef<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_poly_existential_predicates(v))
-    }
-}
-
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::Const<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_const_list(v))
-    }
-}
-
 impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Pattern<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
@@ -363,6 +345,11 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Pattern<'tcx> {
         let pat = (*self).clone().try_fold_with(folder)?;
         Ok(if pat == *self { self } else { folder.cx().mk_pat(pat) })
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        let pat = (*self).clone().fold_with(folder);
+        if pat == *self { self } else { folder.cx().mk_pat(pat) }
+    }
 }
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for Pattern<'tcx> {
@@ -378,6 +365,10 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Ty<'tcx> {
     ) -> Result<Self, F::Error> {
         folder.try_fold_ty(self)
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        folder.fold_ty(self)
+    }
 }
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for Ty<'tcx> {
@@ -436,6 +427,45 @@ impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for Ty<'tcx> {
 
         Ok(if *self.kind() == kind { self } else { folder.cx().mk_ty_from_kind(kind) })
     }
+
+    fn super_fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        let kind = match *self.kind() {
+            ty::RawPtr(ty, mutbl) => ty::RawPtr(ty.fold_with(folder), mutbl),
+            ty::Array(typ, sz) => ty::Array(typ.fold_with(folder), sz.fold_with(folder)),
+            ty::Slice(typ) => ty::Slice(typ.fold_with(folder)),
+            ty::Adt(tid, args) => ty::Adt(tid, args.fold_with(folder)),
+            ty::Dynamic(trait_ty, region, representation) => {
+                ty::Dynamic(trait_ty.fold_with(folder), region.fold_with(folder), representation)
+            }
+            ty::Tuple(ts) => ty::Tuple(ts.fold_with(folder)),
+            ty::FnDef(def_id, args) => ty::FnDef(def_id, args.fold_with(folder)),
+            ty::FnPtr(sig_tys, hdr) => ty::FnPtr(sig_tys.fold_with(folder), hdr),
+            ty::UnsafeBinder(f) => ty::UnsafeBinder(f.fold_with(folder)),
+            ty::Ref(r, ty, mutbl) => ty::Ref(r.fold_with(folder), ty.fold_with(folder), mutbl),
+            ty::Coroutine(did, args) => ty::Coroutine(did, args.fold_with(folder)),
+            ty::CoroutineWitness(did, args) => ty::CoroutineWitness(did, args.fold_with(folder)),
+            ty::Closure(did, args) => ty::Closure(did, args.fold_with(folder)),
+            ty::CoroutineClosure(did, args) => ty::CoroutineClosure(did, args.fold_with(folder)),
+            ty::Alias(kind, data) => ty::Alias(kind, data.fold_with(folder)),
+            ty::Pat(ty, pat) => ty::Pat(ty.fold_with(folder), pat.fold_with(folder)),
+
+            ty::Bool
+            | ty::Char
+            | ty::Str
+            | ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Error(_)
+            | ty::Infer(_)
+            | ty::Param(..)
+            | ty::Bound(..)
+            | ty::Placeholder(..)
+            | ty::Never
+            | ty::Foreign(..) => return self,
+        };
+
+        if *self.kind() == kind { self } else { folder.cx().mk_ty_from_kind(kind) }
+    }
 }
 
 impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for Ty<'tcx> {
@@ -496,6 +526,10 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Region<'tcx> {
     ) -> Result<Self, F::Error> {
         folder.try_fold_region(self)
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        folder.fold_region(self)
+    }
 }
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Region<'tcx> {
@@ -511,6 +545,10 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
     ) -> Result<Self, F::Error> {
         folder.try_fold_predicate(self)
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        folder.fold_predicate(self)
+    }
 }
 
 // FIXME(clause): This is wonky
@@ -521,6 +559,10 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Clause<'tcx> {
     ) -> Result<Self, F::Error> {
         Ok(folder.try_fold_predicate(self.as_predicate())?.expect_clause())
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        folder.fold_predicate(self.as_predicate()).expect_clause()
+    }
 }
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
@@ -543,6 +585,11 @@ impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
         let new = self.kind().try_fold_with(folder)?;
         Ok(folder.cx().reuse_or_mk_predicate(self, new))
     }
+
+    fn super_fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        let new = self.kind().fold_with(folder);
+        folder.cx().reuse_or_mk_predicate(self, new)
+    }
 }
 
 impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
@@ -563,15 +610,6 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_clauses(v))
-    }
-}
-
 impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Const<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
@@ -579,6 +617,10 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Const<'tcx> {
     ) -> Result<Self, F::Error> {
         folder.try_fold_const(self)
     }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        folder.fold_const(self)
+    }
 }
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Const<'tcx> {
@@ -606,6 +648,20 @@ impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Const<'tcx> {
         };
         if kind != self.kind() { Ok(folder.cx().mk_ct_from_kind(kind)) } else { Ok(self) }
     }
+
+    fn super_fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        let kind = match self.kind() {
+            ConstKind::Param(p) => ConstKind::Param(p.fold_with(folder)),
+            ConstKind::Infer(i) => ConstKind::Infer(i.fold_with(folder)),
+            ConstKind::Bound(d, b) => ConstKind::Bound(d.fold_with(folder), b.fold_with(folder)),
+            ConstKind::Placeholder(p) => ConstKind::Placeholder(p.fold_with(folder)),
+            ConstKind::Unevaluated(uv) => ConstKind::Unevaluated(uv.fold_with(folder)),
+            ConstKind::Value(v) => ConstKind::Value(v.fold_with(folder)),
+            ConstKind::Error(e) => ConstKind::Error(e.fold_with(folder)),
+            ConstKind::Expr(e) => ConstKind::Expr(e.fold_with(folder)),
+        };
+        if kind != self.kind() { folder.cx().mk_ct_from_kind(kind) } else { self }
+    }
 }
 
 impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Const<'tcx> {
@@ -639,20 +695,9 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for rustc_span::ErrorGuaranteed {
     ) -> Result<Self, F::Error> {
         Ok(self)
     }
-}
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        _folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        Ok(self)
-    }
-}
-
-impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for InferConst {
-    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, _visitor: &mut V) -> V::Result {
-        V::Result::output()
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, _folder: &mut F) -> Self {
+        self
     }
 }
 
@@ -683,40 +728,50 @@ impl<'tcx, T: TypeFoldable<TyCtxt<'tcx>> + Debug + Clone> TypeFoldable<TyCtxt<'t
             span: self.span.try_fold_with(folder)?,
         })
     }
-}
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx [InlineAsmTemplatePiece] {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        _folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        Ok(self)
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        Spanned { node: self.node.fold_with(folder), span: self.span.fold_with(folder) }
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx [Span] {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<LocalDefId> {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         _folder: &mut F,
     ) -> Result<Self, F::Error> {
         Ok(self)
     }
-}
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<LocalDefId> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        _folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        Ok(self)
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, _folder: &mut F) -> Self {
+        self
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<PlaceElem<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_place_elems(v))
+macro_rules! list_fold {
+    ($($ty:ty : $mk:ident),+ $(,)?) => {
+        $(
+            impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for $ty {
+                fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+                    self,
+                    folder: &mut F,
+                ) -> Result<Self, F::Error> {
+                    ty::util::try_fold_list(self, folder, |tcx, v| tcx.$mk(v))
+                }
+
+                fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(
+                    self,
+                    folder: &mut F,
+                ) -> Self {
+                    ty::util::fold_list(self, folder, |tcx, v| tcx.$mk(v))
+                }
+            }
+        )*
     }
 }
+
+list_fold! {
+    ty::Clauses<'tcx> : mk_clauses,
+    &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> : mk_poly_existential_predicates,
+    &'tcx ty::List<PlaceElem<'tcx>> : mk_place_elems,
+    CanonicalVarInfos<'tcx> : mk_canonical_var_infos,
+}
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 857b462b9eb..e7140720b1c 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -1647,6 +1647,42 @@ pub fn fold_list<'tcx, F, L, T>(
     list: L,
     folder: &mut F,
     intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> L,
+) -> L
+where
+    F: TypeFolder<TyCtxt<'tcx>>,
+    L: AsRef<[T]>,
+    T: TypeFoldable<TyCtxt<'tcx>> + PartialEq + Copy,
+{
+    let slice = list.as_ref();
+    let mut iter = slice.iter().copied();
+    // Look for the first element that changed
+    match iter.by_ref().enumerate().find_map(|(i, t)| {
+        let new_t = t.fold_with(folder);
+        if new_t != t { Some((i, new_t)) } else { None }
+    }) {
+        Some((i, new_t)) => {
+            // An element changed, prepare to intern the resulting list
+            let mut new_list = SmallVec::<[_; 8]>::with_capacity(slice.len());
+            new_list.extend_from_slice(&slice[..i]);
+            new_list.push(new_t);
+            for t in iter {
+                new_list.push(t.fold_with(folder))
+            }
+            intern(folder.cx(), &new_list)
+        }
+        None => list,
+    }
+}
+
+/// Does the equivalent of
+/// ```ignore (illustrative)
+/// let v = self.iter().map(|p| p.try_fold_with(folder)).collect::<SmallVec<[_; 8]>>();
+/// folder.tcx().intern_*(&v)
+/// ```
+pub fn try_fold_list<'tcx, F, L, T>(
+    list: L,
+    folder: &mut F,
+    intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> L,
 ) -> Result<L, F::Error>
 where
     F: FallibleTypeFolder<TyCtxt<'tcx>>,
diff --git a/compiler/rustc_type_ir/src/binder.rs b/compiler/rustc_type_ir/src/binder.rs
index e9055940310..27ea4e211fe 100644
--- a/compiler/rustc_type_ir/src/binder.rs
+++ b/compiler/rustc_type_ir/src/binder.rs
@@ -122,6 +122,10 @@ impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Binder<I, T> {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
         folder.try_fold_binder(self)
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        folder.fold_binder(self)
+    }
 }
 
 impl<I: Interner, T: TypeVisitable<I>> TypeVisitable<I> for Binder<I, T> {
@@ -135,7 +139,11 @@ impl<I: Interner, T: TypeFoldable<I>> TypeSuperFoldable<I> for Binder<I, T> {
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
-        self.try_map_bound(|ty| ty.try_fold_with(folder))
+        self.try_map_bound(|t| t.try_fold_with(folder))
+    }
+
+    fn super_fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        self.map_bound(|t| t.fold_with(folder))
     }
 }
 
diff --git a/compiler/rustc_type_ir/src/fold.rs b/compiler/rustc_type_ir/src/fold.rs
index e58f25f4ce7..ce1188070ca 100644
--- a/compiler/rustc_type_ir/src/fold.rs
+++ b/compiler/rustc_type_ir/src/fold.rs
@@ -45,6 +45,7 @@
 //! - u.fold_with(folder)
 //! ```
 
+use std::convert::Infallible;
 use std::mem;
 use std::sync::Arc;
 
@@ -56,12 +57,6 @@ use crate::inherent::*;
 use crate::visit::{TypeVisitable, TypeVisitableExt as _};
 use crate::{self as ty, Interner, TypeFlags};
 
-#[cfg(feature = "nightly")]
-type Never = !;
-
-#[cfg(not(feature = "nightly"))]
-type Never = std::convert::Infallible;
-
 /// This trait is implemented for every type that can be folded,
 /// providing the skeleton of the traversal.
 ///
@@ -82,18 +77,24 @@ pub trait TypeFoldable<I: Interner>: TypeVisitable<I> + Clone {
     ///
     /// For types of interest (such as `Ty`), the implementation of this method
     /// calls a folder method specifically for that type (such as
-    /// `F::try_fold_ty`). This is where control transfers from `TypeFoldable`
-    /// to `TypeFolder`.
+    /// `F::try_fold_ty`). This is where control transfers from [`TypeFoldable`]
+    /// to [`FallibleTypeFolder`].
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error>;
 
-    /// A convenient alternative to `try_fold_with` for use with infallible
-    /// folders. Do not override this method, to ensure coherence with
-    /// `try_fold_with`.
-    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
-        match self.try_fold_with(folder) {
-            Ok(t) => t,
-        }
-    }
+    /// The entry point for folding. To fold a value `t` with a folder `f`
+    /// call: `t.fold_with(f)`.
+    ///
+    /// For most types, this just traverses the value, calling `fold_with`
+    /// on each field/element.
+    ///
+    /// For types of interest (such as `Ty`), the implementation of this method
+    /// calls a folder method specifically for that type (such as
+    /// `F::fold_ty`). This is where control transfers from `TypeFoldable`
+    /// to `TypeFolder`.
+    ///
+    /// Same as [`TypeFoldable::try_fold_with`], but not fallible. Make sure to keep
+    /// the behavior in sync across functions.
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self;
 }
 
 // This trait is implemented for types of interest.
@@ -112,11 +113,7 @@ pub trait TypeSuperFoldable<I: Interner>: TypeFoldable<I> {
     /// A convenient alternative to `try_super_fold_with` for use with
     /// infallible folders. Do not override this method, to ensure coherence
     /// with `try_super_fold_with`.
-    fn super_fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
-        match self.try_super_fold_with(folder) {
-            Ok(t) => t,
-        }
-    }
+    fn super_fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self;
 }
 
 /// This trait is implemented for every infallible folding traversal. There is
@@ -128,7 +125,7 @@ pub trait TypeSuperFoldable<I: Interner>: TypeFoldable<I> {
 /// A blanket implementation of [`FallibleTypeFolder`] will defer to
 /// the infallible methods of this trait to ensure that the two APIs
 /// are coherent.
-pub trait TypeFolder<I: Interner>: FallibleTypeFolder<I, Error = Never> {
+pub trait TypeFolder<I: Interner>: Sized {
     fn cx(&self) -> I;
 
     fn fold_binder<T>(&mut self, t: ty::Binder<I, T>) -> ty::Binder<I, T>
@@ -195,42 +192,6 @@ pub trait FallibleTypeFolder<I: Interner>: Sized {
     }
 }
 
-// This blanket implementation of the fallible trait for infallible folders
-// delegates to infallible methods to ensure coherence.
-impl<I: Interner, F> FallibleTypeFolder<I> for F
-where
-    F: TypeFolder<I>,
-{
-    type Error = Never;
-
-    fn cx(&self) -> I {
-        TypeFolder::cx(self)
-    }
-
-    fn try_fold_binder<T>(&mut self, t: ty::Binder<I, T>) -> Result<ty::Binder<I, T>, Never>
-    where
-        T: TypeFoldable<I>,
-    {
-        Ok(self.fold_binder(t))
-    }
-
-    fn try_fold_ty(&mut self, t: I::Ty) -> Result<I::Ty, Never> {
-        Ok(self.fold_ty(t))
-    }
-
-    fn try_fold_region(&mut self, r: I::Region) -> Result<I::Region, Never> {
-        Ok(self.fold_region(r))
-    }
-
-    fn try_fold_const(&mut self, c: I::Const) -> Result<I::Const, Never> {
-        Ok(self.fold_const(c))
-    }
-
-    fn try_fold_predicate(&mut self, p: I::Predicate) -> Result<I::Predicate, Never> {
-        Ok(self.fold_predicate(p))
-    }
-}
-
 ///////////////////////////////////////////////////////////////////////////
 // Traversal implementations.
 
@@ -238,6 +199,10 @@ impl<I: Interner, T: TypeFoldable<I>, U: TypeFoldable<I>> TypeFoldable<I> for (T
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<(T, U), F::Error> {
         Ok((self.0.try_fold_with(folder)?, self.1.try_fold_with(folder)?))
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        (self.0.fold_with(folder), self.1.fold_with(folder))
+    }
 }
 
 impl<I: Interner, A: TypeFoldable<I>, B: TypeFoldable<I>, C: TypeFoldable<I>> TypeFoldable<I>
@@ -253,6 +218,10 @@ impl<I: Interner, A: TypeFoldable<I>, B: TypeFoldable<I>, C: TypeFoldable<I>> Ty
             self.2.try_fold_with(folder)?,
         ))
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        (self.0.fold_with(folder), self.1.fold_with(folder), self.2.fold_with(folder))
+    }
 }
 
 impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Option<T> {
@@ -262,6 +231,10 @@ impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Option<T> {
             None => None,
         })
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        Some(self?.fold_with(folder))
+    }
 }
 
 impl<I: Interner, T: TypeFoldable<I>, E: TypeFoldable<I>> TypeFoldable<I> for Result<T, E> {
@@ -271,41 +244,61 @@ impl<I: Interner, T: TypeFoldable<I>, E: TypeFoldable<I>> TypeFoldable<I> for Re
             Err(e) => Err(e.try_fold_with(folder)?),
         })
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        match self {
+            Ok(v) => Ok(v.fold_with(folder)),
+            Err(e) => Err(e.fold_with(folder)),
+        }
+    }
+}
+
+fn fold_arc<T: Clone, E>(
+    mut arc: Arc<T>,
+    fold: impl FnOnce(T) -> Result<T, E>,
+) -> Result<Arc<T>, E> {
+    // We merely want to replace the contained `T`, if at all possible,
+    // so that we don't needlessly allocate a new `Arc` or indeed clone
+    // the contained type.
+    unsafe {
+        // First step is to ensure that we have a unique reference to
+        // the contained type, which `Arc::make_mut` will accomplish (by
+        // allocating a new `Arc` and cloning the `T` only if required).
+        // This is done *before* casting to `Arc<ManuallyDrop<T>>` so that
+        // panicking during `make_mut` does not leak the `T`.
+        Arc::make_mut(&mut arc);
+
+        // Casting to `Arc<ManuallyDrop<T>>` is safe because `ManuallyDrop`
+        // is `repr(transparent)`.
+        let ptr = Arc::into_raw(arc).cast::<mem::ManuallyDrop<T>>();
+        let mut unique = Arc::from_raw(ptr);
+
+        // Call to `Arc::make_mut` above guarantees that `unique` is the
+        // sole reference to the contained value, so we can avoid doing
+        // a checked `get_mut` here.
+        let slot = Arc::get_mut(&mut unique).unwrap_unchecked();
+
+        // Semantically move the contained type out from `unique`, fold
+        // it, then move the folded value back into `unique`. Should
+        // folding fail, `ManuallyDrop` ensures that the "moved-out"
+        // value is not re-dropped.
+        let owned = mem::ManuallyDrop::take(slot);
+        let folded = fold(owned)?;
+        *slot = mem::ManuallyDrop::new(folded);
+
+        // Cast back to `Arc<T>`.
+        Ok(Arc::from_raw(Arc::into_raw(unique).cast()))
+    }
 }
 
 impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Arc<T> {
-    fn try_fold_with<F: FallibleTypeFolder<I>>(mut self, folder: &mut F) -> Result<Self, F::Error> {
-        // We merely want to replace the contained `T`, if at all possible,
-        // so that we don't needlessly allocate a new `Arc` or indeed clone
-        // the contained type.
-        unsafe {
-            // First step is to ensure that we have a unique reference to
-            // the contained type, which `Arc::make_mut` will accomplish (by
-            // allocating a new `Arc` and cloning the `T` only if required).
-            // This is done *before* casting to `Arc<ManuallyDrop<T>>` so that
-            // panicking during `make_mut` does not leak the `T`.
-            Arc::make_mut(&mut self);
-
-            // Casting to `Arc<ManuallyDrop<T>>` is safe because `ManuallyDrop`
-            // is `repr(transparent)`.
-            let ptr = Arc::into_raw(self).cast::<mem::ManuallyDrop<T>>();
-            let mut unique = Arc::from_raw(ptr);
-
-            // Call to `Arc::make_mut` above guarantees that `unique` is the
-            // sole reference to the contained value, so we can avoid doing
-            // a checked `get_mut` here.
-            let slot = Arc::get_mut(&mut unique).unwrap_unchecked();
-
-            // Semantically move the contained type out from `unique`, fold
-            // it, then move the folded value back into `unique`. Should
-            // folding fail, `ManuallyDrop` ensures that the "moved-out"
-            // value is not re-dropped.
-            let owned = mem::ManuallyDrop::take(slot);
-            let folded = owned.try_fold_with(folder)?;
-            *slot = mem::ManuallyDrop::new(folded);
-
-            // Cast back to `Arc<T>`.
-            Ok(Arc::from_raw(Arc::into_raw(unique).cast()))
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        fold_arc(self, |t| t.try_fold_with(folder))
+    }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        match fold_arc::<T, Infallible>(self, |t| Ok(t.fold_with(folder))) {
+            Ok(t) => t,
         }
     }
 }
@@ -315,30 +308,51 @@ impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Box<T> {
         *self = (*self).try_fold_with(folder)?;
         Ok(self)
     }
+
+    fn fold_with<F: TypeFolder<I>>(mut self, folder: &mut F) -> Self {
+        *self = (*self).fold_with(folder);
+        self
+    }
 }
 
 impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Vec<T> {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.into_iter().map(|t| t.try_fold_with(folder)).collect()
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        self.into_iter().map(|t| t.fold_with(folder)).collect()
+    }
 }
 
 impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for ThinVec<T> {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.into_iter().map(|t| t.try_fold_with(folder)).collect()
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        self.into_iter().map(|t| t.fold_with(folder)).collect()
+    }
 }
 
 impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Box<[T]> {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
         Vec::from(self).try_fold_with(folder).map(Vec::into_boxed_slice)
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        Vec::into_boxed_slice(Vec::from(self).fold_with(folder))
+    }
 }
 
 impl<I: Interner, T: TypeFoldable<I>, Ix: Idx> TypeFoldable<I> for IndexVec<Ix, T> {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.raw.try_fold_with(folder).map(IndexVec::from_raw)
     }
+
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        IndexVec::from_raw(self.raw.fold_with(folder))
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
diff --git a/compiler/rustc_type_ir_macros/src/lib.rs b/compiler/rustc_type_ir_macros/src/lib.rs
index 8eefecdc980..3a10d0d41ef 100644
--- a/compiler/rustc_type_ir_macros/src/lib.rs
+++ b/compiler/rustc_type_ir_macros/src/lib.rs
@@ -83,7 +83,7 @@ fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::Toke
     s.add_where_predicate(parse_quote! { I: Interner });
     s.add_bounds(synstructure::AddBounds::Fields);
     s.bind_with(|_| synstructure::BindStyle::Move);
-    let body_fold = s.each_variant(|vi| {
+    let body_try_fold = s.each_variant(|vi| {
         let bindings = vi.bindings();
         vi.construct(|_, index| {
             let bind = &bindings[index];
@@ -99,6 +99,22 @@ fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::Toke
         })
     });
 
+    let body_fold = s.each_variant(|vi| {
+        let bindings = vi.bindings();
+        vi.construct(|_, index| {
+            let bind = &bindings[index];
+
+            // retain value of fields with #[type_foldable(identity)]
+            if has_ignore_attr(&bind.ast().attrs, "type_foldable", "identity") {
+                bind.to_token_stream()
+            } else {
+                quote! {
+                    ::rustc_type_ir::TypeFoldable::fold_with(#bind, __folder)
+                }
+            }
+        })
+    });
+
     // We filter fields which get ignored and don't require them to implement
     // `TypeFoldable`. We do so after generating `body_fold` as we still need
     // to generate code for them.
@@ -111,7 +127,14 @@ fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::Toke
                 self,
                 __folder: &mut __F
             ) -> Result<Self, __F::Error> {
-                Ok(match self { #body_fold })
+                Ok(match self { #body_try_fold })
+            }
+
+            fn fold_with<__F: ::rustc_type_ir::TypeFolder<I>>(
+                self,
+                __folder: &mut __F
+            ) -> Self {
+                match self { #body_fold }
             }
         },
     )