Auto merge of #73180 - matthewjasper:predicate-cache, r=nikomatsakis

Cache flags and escaping vars for predicates

With predicates becoming interned (rust-lang/compiler-team#285) this is now possible and could be a perf win. It would become an even larger win once we have recursive predicates.

cc @lcnr @nikomatsakis

r? @ghost

11 files changed, 321 insertions, 100 deletions

diff --git a/src/librustc_metadata/rmeta/decoder.rs b/src/librustc_metadata/rmeta/decoder.rs
index 800f794121a..2254d553337 100644
--- a/src/librustc_metadata/rmeta/decoder.rs
+++ b/src/librustc_metadata/rmeta/decoder.rs
@@ -294,15 +294,36 @@ impl<'a, 'tcx> TyDecoder<'tcx> for DecodeContext<'a, 'tcx> {
 
         let key = ty::CReaderCacheKey { cnum: self.cdata().cnum, pos: shorthand };
 
-        if let Some(&ty) = tcx.rcache.borrow().get(&key) {
+        if let Some(&ty) = tcx.ty_rcache.borrow().get(&key) {
             return Ok(ty);
         }
 
         let ty = or_insert_with(self)?;
-        tcx.rcache.borrow_mut().insert(key, ty);
+        tcx.ty_rcache.borrow_mut().insert(key, ty);
         Ok(ty)
     }
 
+    fn cached_predicate_for_shorthand<F>(
+        &mut self,
+        shorthand: usize,
+        or_insert_with: F,
+    ) -> Result<ty::Predicate<'tcx>, Self::Error>
+    where
+        F: FnOnce(&mut Self) -> Result<ty::Predicate<'tcx>, Self::Error>,
+    {
+        let tcx = self.tcx();
+
+        let key = ty::CReaderCacheKey { cnum: self.cdata().cnum, pos: shorthand };
+
+        if let Some(&pred) = tcx.pred_rcache.borrow().get(&key) {
+            return Ok(pred);
+        }
+
+        let pred = or_insert_with(self)?;
+        tcx.pred_rcache.borrow_mut().insert(key, pred);
+        Ok(pred)
+    }
+
     fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
     where
         F: FnOnce(&mut Self) -> R,
diff --git a/src/librustc_metadata/rmeta/encoder.rs b/src/librustc_metadata/rmeta/encoder.rs
index 31821ea459f..d01c767e2bc 100644
--- a/src/librustc_metadata/rmeta/encoder.rs
+++ b/src/librustc_metadata/rmeta/encoder.rs
@@ -239,6 +239,17 @@ where
     }
 }
 
+impl<'b, 'tcx> SpecializedEncoder<ty::Predicate<'b>> for EncodeContext<'tcx> {
+    fn specialized_encode(&mut self, predicate: &ty::Predicate<'b>) -> Result<(), Self::Error> {
+        debug_assert!(self.tcx.lift(predicate).is_some());
+        let predicate =
+            unsafe { std::mem::transmute::<&ty::Predicate<'b>, &ty::Predicate<'tcx>>(predicate) };
+        ty_codec::encode_with_shorthand(self, predicate, |encoder| {
+            &mut encoder.predicate_shorthands
+        })
+    }
+}
+
 impl<'tcx> SpecializedEncoder<interpret::AllocId> for EncodeContext<'tcx> {
     fn specialized_encode(&mut self, alloc_id: &interpret::AllocId) -> Result<(), Self::Error> {
         use std::collections::hash_map::Entry;
@@ -256,22 +267,6 @@ impl<'tcx> SpecializedEncoder<interpret::AllocId> for EncodeContext<'tcx> {
     }
 }
 
-impl<'a, 'b, 'tcx> SpecializedEncoder<&'a [(ty::Predicate<'b>, Span)]> for EncodeContext<'tcx> {
-    fn specialized_encode(
-        &mut self,
-        predicates: &&'a [(ty::Predicate<'b>, Span)],
-    ) -> Result<(), Self::Error> {
-        debug_assert!(self.tcx.lift(*predicates).is_some());
-        let predicates = unsafe {
-            std::mem::transmute::<
-                &&'a [(ty::Predicate<'b>, Span)],
-                &&'tcx [(ty::Predicate<'tcx>, Span)],
-            >(predicates)
-        };
-        ty_codec::encode_spanned_predicates(self, &predicates, |ecx| &mut ecx.predicate_shorthands)
-    }
-}
-
 impl<'tcx> SpecializedEncoder<Fingerprint> for EncodeContext<'tcx> {
     fn specialized_encode(&mut self, f: &Fingerprint) -> Result<(), Self::Error> {
         f.encode_opaque(&mut self.opaque)
diff --git a/src/librustc_middle/arena.rs b/src/librustc_middle/arena.rs
index aaef9871aa5..4f1889aeb16 100644
--- a/src/librustc_middle/arena.rs
+++ b/src/librustc_middle/arena.rs
@@ -100,6 +100,7 @@ macro_rules! arena_types {
 
             // Interned types
             [] tys: rustc_middle::ty::TyS<$tcx>, rustc_middle::ty::TyS<'_x>;
+            [] predicates: rustc_middle::ty::PredicateInner<$tcx>, rustc_middle::ty::PredicateInner<'_x>;
 
             // HIR query types
             [few] indexed_hir: rustc_middle::hir::map::IndexedHir<$tcx>, rustc_middle::hir::map::IndexedHir<'_x>;
diff --git a/src/librustc_middle/ty/codec.rs b/src/librustc_middle/ty/codec.rs
index 1a8e5c45dd2..67ceaca103e 100644
--- a/src/librustc_middle/ty/codec.rs
+++ b/src/librustc_middle/ty/codec.rs
@@ -10,7 +10,7 @@ use crate::arena::ArenaAllocatable;
 use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
 use crate::mir::{self, interpret::Allocation};
 use crate::ty::subst::SubstsRef;
-use crate::ty::{self, List, ToPredicate, Ty, TyCtxt};
+use crate::ty::{self, List, Ty, TyCtxt};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::{CrateNum, DefId};
 use rustc_serialize::{opaque, Decodable, Decoder, Encodable, Encoder};
@@ -95,23 +95,6 @@ where
     Ok(())
 }
 
-pub fn encode_spanned_predicates<'tcx, E, C>(
-    encoder: &mut E,
-    predicates: &[(ty::Predicate<'tcx>, Span)],
-    cache: C,
-) -> Result<(), E::Error>
-where
-    E: TyEncoder,
-    C: for<'b> Fn(&'b mut E) -> &'b mut FxHashMap<ty::Predicate<'tcx>, usize>,
-{
-    predicates.len().encode(encoder)?;
-    for (predicate, span) in predicates {
-        encode_with_shorthand(encoder, predicate, &cache)?;
-        span.encode(encoder)?;
-    }
-    Ok(())
-}
-
 pub trait TyDecoder<'tcx>: Decoder {
     fn tcx(&self) -> TyCtxt<'tcx>;
 
@@ -127,6 +110,14 @@ pub trait TyDecoder<'tcx>: Decoder {
     where
         F: FnOnce(&mut Self) -> Result<Ty<'tcx>, Self::Error>;
 
+    fn cached_predicate_for_shorthand<F>(
+        &mut self,
+        shorthand: usize,
+        or_insert_with: F,
+    ) -> Result<ty::Predicate<'tcx>, Self::Error>
+    where
+        F: FnOnce(&mut Self) -> Result<ty::Predicate<'tcx>, Self::Error>;
+
     fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
     where
         F: FnOnce(&mut Self) -> R;
@@ -189,6 +180,26 @@ where
 }
 
 #[inline]
+pub fn decode_predicate<D>(decoder: &mut D) -> Result<ty::Predicate<'tcx>, D::Error>
+where
+    D: TyDecoder<'tcx>,
+{
+    // Handle shorthands first, if we have an usize > 0x80.
+    if decoder.positioned_at_shorthand() {
+        let pos = decoder.read_usize()?;
+        assert!(pos >= SHORTHAND_OFFSET);
+        let shorthand = pos - SHORTHAND_OFFSET;
+
+        decoder.cached_predicate_for_shorthand(shorthand, |decoder| {
+            decoder.with_position(shorthand, ty::Predicate::decode)
+        })
+    } else {
+        let tcx = decoder.tcx();
+        Ok(tcx.mk_predicate(ty::PredicateKind::decode(decoder)?))
+    }
+}
+
+#[inline]
 pub fn decode_spanned_predicates<D>(
     decoder: &mut D,
 ) -> Result<&'tcx [(ty::Predicate<'tcx>, Span)], D::Error>
@@ -198,20 +209,7 @@ where
     let tcx = decoder.tcx();
     Ok(tcx.arena.alloc_from_iter(
         (0..decoder.read_usize()?)
-            .map(|_| {
-                // Handle shorthands first, if we have an usize > 0x80.
-                let predicate_kind = if decoder.positioned_at_shorthand() {
-                    let pos = decoder.read_usize()?;
-                    assert!(pos >= SHORTHAND_OFFSET);
-                    let shorthand = pos - SHORTHAND_OFFSET;
-
-                    decoder.with_position(shorthand, ty::PredicateKind::decode)
-                } else {
-                    ty::PredicateKind::decode(decoder)
-                }?;
-                let predicate = predicate_kind.to_predicate(tcx);
-                Ok((predicate, Decodable::decode(decoder)?))
-            })
+            .map(|_| Decodable::decode(decoder))
             .collect::<Result<Vec<_>, _>>()?,
     ))
 }
@@ -421,7 +419,6 @@ macro_rules! implement_ty_decoder {
             // FIXME(#36588): These impls are horribly unsound as they allow
             // the caller to pick any lifetime for `'tcx`, including `'static`.
 
-            rustc_hir::arena_types!(impl_arena_allocatable_decoders, [$DecoderName [$($typaram),*]], 'tcx);
             arena_types!(impl_arena_allocatable_decoders, [$DecoderName [$($typaram),*]], 'tcx);
 
             impl<$($typaram),*> SpecializedDecoder<CrateNum>
@@ -436,7 +433,24 @@ macro_rules! implement_ty_decoder {
             where &'_x ty::TyS<'_y>: UseSpecializedDecodable
             {
                 fn specialized_decode(&mut self) -> Result<&'_x ty::TyS<'_y>, Self::Error> {
-                    unsafe { transmute::<Result<ty::Ty<'tcx>, Self::Error>, Result<&'_x ty::TyS<'_y>, Self::Error>>(decode_ty(self)) }
+                    unsafe {
+                        transmute::<
+                            Result<ty::Ty<'tcx>, Self::Error>,
+                            Result<&'_x ty::TyS<'_y>, Self::Error>,
+                        >(decode_ty(self))
+                    }
+                }
+            }
+
+            impl<'_x, $($typaram),*> SpecializedDecoder<ty::Predicate<'_x>>
+            for $DecoderName<$($typaram),*> {
+                fn specialized_decode(&mut self) -> Result<ty::Predicate<'_x>, Self::Error> {
+                    unsafe {
+                        transmute::<
+                            Result<ty::Predicate<'tcx>, Self::Error>,
+                            Result<ty::Predicate<'_x>, Self::Error>,
+                        >(decode_predicate(self))
+                    }
                 }
             }
 
diff --git a/src/librustc_middle/ty/context.rs b/src/librustc_middle/ty/context.rs
index 5b44ffe8caf..56f4ae9e984 100644
--- a/src/librustc_middle/ty/context.rs
+++ b/src/librustc_middle/ty/context.rs
@@ -19,8 +19,9 @@ use crate::ty::TyKind::*;
 use crate::ty::{
     self, query, AdtDef, AdtKind, BindingMode, BoundVar, CanonicalPolyFnSig, Const, ConstVid,
     DefIdTree, ExistentialPredicate, FloatVar, FloatVid, GenericParamDefKind, InferConst, InferTy,
-    IntVar, IntVid, List, ParamConst, ParamTy, PolyFnSig, Predicate, PredicateKind, ProjectionTy,
-    Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar, TyVid, TypeAndMut,
+    IntVar, IntVid, List, ParamConst, ParamTy, PolyFnSig, Predicate, PredicateInner, PredicateKind,
+    ProjectionTy, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar,
+    TyVid, TypeAndMut,
 };
 use rustc_ast::ast;
 use rustc_ast::expand::allocator::AllocatorKind;
@@ -76,7 +77,7 @@ pub struct CtxtInterners<'tcx> {
     canonical_var_infos: InternedSet<'tcx, List<CanonicalVarInfo>>,
     region: InternedSet<'tcx, RegionKind>,
     existential_predicates: InternedSet<'tcx, List<ExistentialPredicate<'tcx>>>,
-    predicate_kind: InternedSet<'tcx, PredicateKind<'tcx>>,
+    predicate: InternedSet<'tcx, PredicateInner<'tcx>>,
     predicates: InternedSet<'tcx, List<Predicate<'tcx>>>,
     projs: InternedSet<'tcx, List<ProjectionKind>>,
     place_elems: InternedSet<'tcx, List<PlaceElem<'tcx>>>,
@@ -95,7 +96,7 @@ impl<'tcx> CtxtInterners<'tcx> {
             region: Default::default(),
             existential_predicates: Default::default(),
             canonical_var_infos: Default::default(),
-            predicate_kind: Default::default(),
+            predicate: Default::default(),
             predicates: Default::default(),
             projs: Default::default(),
             place_elems: Default::default(),
@@ -123,6 +124,23 @@ impl<'tcx> CtxtInterners<'tcx> {
             })
             .0
     }
+
+    #[inline(never)]
+    fn intern_predicate(&self, kind: PredicateKind<'tcx>) -> &'tcx PredicateInner<'tcx> {
+        self.predicate
+            .intern(kind, |kind| {
+                let flags = super::flags::FlagComputation::for_predicate(&kind);
+
+                let predicate_struct = PredicateInner {
+                    kind,
+                    flags: flags.flags,
+                    outer_exclusive_binder: flags.outer_exclusive_binder,
+                };
+
+                Interned(self.arena.alloc(predicate_struct))
+            })
+            .0
+    }
 }
 
 pub struct CommonTypes<'tcx> {
@@ -938,8 +956,9 @@ pub struct GlobalCtxt<'tcx> {
     /// via `extern crate` item and not `--extern` option or compiler built-in.
     pub extern_prelude: FxHashMap<Symbol, bool>,
 
-    // Internal cache for metadata decoding. No need to track deps on this.
-    pub rcache: Lock<FxHashMap<ty::CReaderCacheKey, Ty<'tcx>>>,
+    // Internal caches for metadata decoding. No need to track deps on this.
+    pub ty_rcache: Lock<FxHashMap<ty::CReaderCacheKey, Ty<'tcx>>>,
+    pub pred_rcache: Lock<FxHashMap<ty::CReaderCacheKey, Predicate<'tcx>>>,
 
     /// Caches the results of trait selection. This cache is used
     /// for things that do not have to do with the parameters in scope.
@@ -1128,7 +1147,8 @@ impl<'tcx> TyCtxt<'tcx> {
             definitions,
             def_path_hash_to_def_id,
             queries: query::Queries::new(providers, extern_providers, on_disk_query_result_cache),
-            rcache: Default::default(),
+            ty_rcache: Default::default(),
+            pred_rcache: Default::default(),
             selection_cache: Default::default(),
             evaluation_cache: Default::default(),
             crate_name: Symbol::intern(crate_name),
@@ -1625,7 +1645,7 @@ macro_rules! nop_list_lift {
 nop_lift! {type_; Ty<'a> => Ty<'tcx>}
 nop_lift! {region; Region<'a> => Region<'tcx>}
 nop_lift! {const_; &'a Const<'a> => &'tcx Const<'tcx>}
-nop_lift! {predicate_kind; &'a PredicateKind<'a> => &'tcx PredicateKind<'tcx>}
+nop_lift! {predicate; &'a PredicateInner<'a> => &'tcx PredicateInner<'tcx>}
 
 nop_list_lift! {type_list; Ty<'a> => Ty<'tcx>}
 nop_list_lift! {existential_predicates; ExistentialPredicate<'a> => ExistentialPredicate<'tcx>}
@@ -1984,6 +2004,26 @@ impl<'tcx> Borrow<TyKind<'tcx>> for Interned<'tcx, TyS<'tcx>> {
         &self.0.kind
     }
 }
+// N.B., an `Interned<PredicateInner>` compares and hashes as a `PredicateKind`.
+impl<'tcx> PartialEq for Interned<'tcx, PredicateInner<'tcx>> {
+    fn eq(&self, other: &Interned<'tcx, PredicateInner<'tcx>>) -> bool {
+        self.0.kind == other.0.kind
+    }
+}
+
+impl<'tcx> Eq for Interned<'tcx, PredicateInner<'tcx>> {}
+
+impl<'tcx> Hash for Interned<'tcx, PredicateInner<'tcx>> {
+    fn hash<H: Hasher>(&self, s: &mut H) {
+        self.0.kind.hash(s)
+    }
+}
+
+impl<'tcx> Borrow<PredicateKind<'tcx>> for Interned<'tcx, PredicateInner<'tcx>> {
+    fn borrow<'a>(&'a self) -> &'a PredicateKind<'tcx> {
+        &self.0.kind
+    }
+}
 
 // N.B., an `Interned<List<T>>` compares and hashes as its elements.
 impl<'tcx, T: PartialEq> PartialEq for Interned<'tcx, List<T>> {
@@ -2050,11 +2090,10 @@ macro_rules! direct_interners {
     }
 }
 
-direct_interners!(
+direct_interners! {
     region: mk_region(RegionKind),
     const_: mk_const(Const<'tcx>),
-    predicate_kind: intern_predicate_kind(PredicateKind<'tcx>),
-);
+}
 
 macro_rules! slice_interners {
     ($($field:ident: $method:ident($ty:ty)),+) => (
@@ -2125,8 +2164,8 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_predicate(&self, kind: PredicateKind<'tcx>) -> Predicate<'tcx> {
-        let kind = self.intern_predicate_kind(kind);
-        Predicate { kind }
+        let inner = self.interners.intern_predicate(kind);
+        Predicate { inner }
     }
 
     pub fn mk_mach_int(self, tm: ast::IntTy) -> Ty<'tcx> {
diff --git a/src/librustc_middle/ty/flags.rs b/src/librustc_middle/ty/flags.rs
index d13be4f4285..c782eee9387 100644
--- a/src/librustc_middle/ty/flags.rs
+++ b/src/librustc_middle/ty/flags.rs
@@ -1,5 +1,6 @@
 use crate::ty::subst::{GenericArg, GenericArgKind};
 use crate::ty::{self, InferConst, Ty, TypeFlags};
+use std::slice;
 
 #[derive(Debug)]
 pub struct FlagComputation {
@@ -21,6 +22,12 @@ impl FlagComputation {
         result
     }
 
+    pub fn for_predicate(kind: &ty::PredicateKind<'_>) -> FlagComputation {
+        let mut result = FlagComputation::new();
+        result.add_predicate_kind(kind);
+        result
+    }
+
     pub fn for_const(c: &ty::Const<'_>) -> TypeFlags {
         let mut result = FlagComputation::new();
         result.add_const(c);
@@ -32,7 +39,7 @@ impl FlagComputation {
     }
 
     /// indicates that `self` refers to something at binding level `binder`
-    fn add_binder(&mut self, binder: ty::DebruijnIndex) {
+    fn add_bound_var(&mut self, binder: ty::DebruijnIndex) {
         let exclusive_binder = binder.shifted_in(1);
         self.add_exclusive_binder(exclusive_binder);
     }
@@ -46,7 +53,7 @@ impl FlagComputation {
 
     /// Adds the flags/depth from a set of types that appear within the current type, but within a
     /// region binder.
-    fn add_bound_computation(&mut self, computation: &FlagComputation) {
+    fn add_bound_computation(&mut self, computation: FlagComputation) {
         self.add_flags(computation.flags);
 
         // The types that contributed to `computation` occurred within
@@ -84,7 +91,7 @@ impl FlagComputation {
             &ty::GeneratorWitness(ref ts) => {
                 let mut computation = FlagComputation::new();
                 computation.add_tys(&ts.skip_binder()[..]);
-                self.add_bound_computation(&computation);
+                self.add_bound_computation(computation);
             }
 
             &ty::Closure(_, ref substs) => {
@@ -92,7 +99,7 @@ impl FlagComputation {
             }
 
             &ty::Bound(debruijn, _) => {
-                self.add_binder(debruijn);
+                self.add_bound_var(debruijn);
             }
 
             &ty::Placeholder(..) => {
@@ -133,12 +140,12 @@ impl FlagComputation {
                         ty::ExistentialPredicate::Projection(p) => {
                             let mut proj_computation = FlagComputation::new();
                             proj_computation.add_existential_projection(&p);
-                            self.add_bound_computation(&proj_computation);
+                            self.add_bound_computation(proj_computation);
                         }
                         ty::ExistentialPredicate::AutoTrait(_) => {}
                     }
                 }
-                self.add_bound_computation(&computation);
+                self.add_bound_computation(computation);
                 self.add_region(r);
             }
 
@@ -172,6 +179,63 @@ impl FlagComputation {
         }
     }
 
+    fn add_predicate_kind(&mut self, kind: &ty::PredicateKind<'_>) {
+        match kind {
+            ty::PredicateKind::Trait(trait_pred, _constness) => {
+                let mut computation = FlagComputation::new();
+                computation.add_substs(trait_pred.skip_binder().trait_ref.substs);
+
+                self.add_bound_computation(computation);
+            }
+            ty::PredicateKind::RegionOutlives(poly_outlives) => {
+                let mut computation = FlagComputation::new();
+                let ty::OutlivesPredicate(a, b) = poly_outlives.skip_binder();
+                computation.add_region(a);
+                computation.add_region(b);
+
+                self.add_bound_computation(computation);
+            }
+            ty::PredicateKind::TypeOutlives(poly_outlives) => {
+                let mut computation = FlagComputation::new();
+                let ty::OutlivesPredicate(ty, region) = poly_outlives.skip_binder();
+                computation.add_ty(ty);
+                computation.add_region(region);
+
+                self.add_bound_computation(computation);
+            }
+            ty::PredicateKind::Subtype(poly_subtype) => {
+                let mut computation = FlagComputation::new();
+                let ty::SubtypePredicate { a_is_expected: _, a, b } = poly_subtype.skip_binder();
+                computation.add_ty(a);
+                computation.add_ty(b);
+
+                self.add_bound_computation(computation);
+            }
+            ty::PredicateKind::Projection(projection) => {
+                let mut computation = FlagComputation::new();
+                let ty::ProjectionPredicate { projection_ty, ty } = projection.skip_binder();
+                computation.add_projection_ty(projection_ty);
+                computation.add_ty(ty);
+
+                self.add_bound_computation(computation);
+            }
+            ty::PredicateKind::WellFormed(arg) => {
+                self.add_substs(slice::from_ref(arg));
+            }
+            ty::PredicateKind::ObjectSafe(_def_id) => {}
+            ty::PredicateKind::ClosureKind(_def_id, substs, _kind) => {
+                self.add_substs(substs);
+            }
+            ty::PredicateKind::ConstEvaluatable(_def_id, substs) => {
+                self.add_substs(substs);
+            }
+            ty::PredicateKind::ConstEquate(expected, found) => {
+                self.add_const(expected);
+                self.add_const(found);
+            }
+        }
+    }
+
     fn add_ty(&mut self, ty: Ty<'_>) {
         self.add_flags(ty.flags);
         self.add_exclusive_binder(ty.outer_exclusive_binder);
@@ -189,13 +253,13 @@ impl FlagComputation {
         computation.add_tys(fn_sig.skip_binder().inputs());
         computation.add_ty(fn_sig.skip_binder().output());
 
-        self.add_bound_computation(&computation);
+        self.add_bound_computation(computation);
     }
 
     fn add_region(&mut self, r: ty::Region<'_>) {
         self.add_flags(r.type_flags());
         if let ty::ReLateBound(debruijn, _) = *r {
-            self.add_binder(debruijn);
+            self.add_bound_var(debruijn);
         }
     }
 
@@ -214,7 +278,7 @@ impl FlagComputation {
                 }
             }
             ty::ConstKind::Bound(debruijn, _) => {
-                self.add_binder(debruijn);
+                self.add_bound_var(debruijn);
             }
             ty::ConstKind::Param(_) => {
                 self.add_flags(TypeFlags::HAS_CT_PARAM);
diff --git a/src/librustc_middle/ty/fold.rs b/src/librustc_middle/ty/fold.rs
index 24dbf7b8c46..2d25c7c6ac9 100644
--- a/src/librustc_middle/ty/fold.rs
+++ b/src/librustc_middle/ty/fold.rs
@@ -31,6 +31,7 @@
 //! These methods return true to indicate that the visitor has found what it is
 //! looking for, and does not need to visit anything else.
 
+use crate::ty::structural_impls::PredicateVisitor;
 use crate::ty::{self, flags::FlagComputation, Binder, Ty, TyCtxt, TypeFlags};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
@@ -908,6 +909,12 @@ impl<'tcx> TypeVisitor<'tcx> for HasEscapingVarsVisitor {
     }
 }
 
+impl<'tcx> PredicateVisitor<'tcx> for HasEscapingVarsVisitor {
+    fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool {
+        predicate.inner.outer_exclusive_binder > self.outer_index
+    }
+}
+
 // FIXME: Optimize for checking for infer flags
 struct HasTypeFlagsVisitor {
     flags: ty::TypeFlags,
@@ -932,6 +939,15 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 }
 
+impl<'tcx> PredicateVisitor<'tcx> for HasTypeFlagsVisitor {
+    fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool {
+        debug!(
+            "HasTypeFlagsVisitor: predicate={:?} predicate.flags={:?} self.flags={:?}",
+            predicate, predicate.inner.flags, self.flags
+        );
+        predicate.inner.flags.intersects(self.flags)
+    }
+}
 /// Collects all the late-bound regions at the innermost binding level
 /// into a hash set.
 struct LateBoundRegionsCollector {
diff --git a/src/librustc_middle/ty/mod.rs b/src/librustc_middle/ty/mod.rs
index 56fb3b58d3f..6b7940ed7ab 100644
--- a/src/librustc_middle/ty/mod.rs
+++ b/src/librustc_middle/ty/mod.rs
@@ -627,7 +627,7 @@ impl<'tcx> Hash for TyS<'tcx> {
     }
 }
 
-impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::TyS<'tcx> {
+impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for TyS<'tcx> {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
         let ty::TyS {
             ref kind,
@@ -1001,16 +1001,35 @@ impl<'tcx> GenericPredicates<'tcx> {
     }
 }
 
-#[derive(Clone, Copy, Hash, RustcEncodable, RustcDecodable, Lift)]
-#[derive(HashStable)]
+#[derive(Debug)]
+crate struct PredicateInner<'tcx> {
+    kind: PredicateKind<'tcx>,
+    flags: TypeFlags,
+    /// See the comment for the corresponding field of [TyS].
+    outer_exclusive_binder: ty::DebruijnIndex,
+}
+
+#[cfg(target_arch = "x86_64")]
+static_assert_size!(PredicateInner<'_>, 40);
+
+#[derive(Clone, Copy, Lift)]
 pub struct Predicate<'tcx> {
-    kind: &'tcx PredicateKind<'tcx>,
+    inner: &'tcx PredicateInner<'tcx>,
 }
 
+impl rustc_serialize::UseSpecializedEncodable for Predicate<'_> {}
+impl rustc_serialize::UseSpecializedDecodable for Predicate<'_> {}
+
 impl<'tcx> PartialEq for Predicate<'tcx> {
     fn eq(&self, other: &Self) -> bool {
         // `self.kind` is always interned.
-        ptr::eq(self.kind, other.kind)
+        ptr::eq(self.inner, other.inner)
+    }
+}
+
+impl Hash for Predicate<'_> {
+    fn hash<H: Hasher>(&self, s: &mut H) {
+        (self.inner as *const PredicateInner<'_>).hash(s)
     }
 }
 
@@ -1019,7 +1038,22 @@ impl<'tcx> Eq for Predicate<'tcx> {}
 impl<'tcx> Predicate<'tcx> {
     #[inline(always)]
     pub fn kind(self) -> &'tcx PredicateKind<'tcx> {
-        self.kind
+        &self.inner.kind
+    }
+}
+
+impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Predicate<'tcx> {
+    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
+        let PredicateInner {
+            ref kind,
+
+            // The other fields just provide fast access to information that is
+            // also contained in `kind`, so no need to hash them.
+            flags: _,
+            outer_exclusive_binder: _,
+        } = self.inner;
+
+        kind.hash_stable(hcx, hasher);
     }
 }
 
diff --git a/src/librustc_middle/ty/query/on_disk_cache.rs b/src/librustc_middle/ty/query/on_disk_cache.rs
index 5374dff4224..c84a7c38d0a 100644
--- a/src/librustc_middle/ty/query/on_disk_cache.rs
+++ b/src/librustc_middle/ty/query/on_disk_cache.rs
@@ -524,16 +524,39 @@ impl<'a, 'tcx> TyDecoder<'tcx> for CacheDecoder<'a, 'tcx> {
         let cache_key =
             ty::CReaderCacheKey { cnum: CrateNum::ReservedForIncrCompCache, pos: shorthand };
 
-        if let Some(&ty) = tcx.rcache.borrow().get(&cache_key) {
+        if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) {
             return Ok(ty);
         }
 
         let ty = or_insert_with(self)?;
         // This may overwrite the entry, but it should overwrite with the same value.
-        tcx.rcache.borrow_mut().insert_same(cache_key, ty);
+        tcx.ty_rcache.borrow_mut().insert_same(cache_key, ty);
         Ok(ty)
     }
 
+    fn cached_predicate_for_shorthand<F>(
+        &mut self,
+        shorthand: usize,
+        or_insert_with: F,
+    ) -> Result<ty::Predicate<'tcx>, Self::Error>
+    where
+        F: FnOnce(&mut Self) -> Result<ty::Predicate<'tcx>, Self::Error>,
+    {
+        let tcx = self.tcx();
+
+        let cache_key =
+            ty::CReaderCacheKey { cnum: CrateNum::ReservedForIncrCompCache, pos: shorthand };
+
+        if let Some(&pred) = tcx.pred_rcache.borrow().get(&cache_key) {
+            return Ok(pred);
+        }
+
+        let pred = or_insert_with(self)?;
+        // This may overwrite the entry, but it should overwrite with the same value.
+        tcx.pred_rcache.borrow_mut().insert_same(cache_key, pred);
+        Ok(pred)
+    }
+
     fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
     where
         F: FnOnce(&mut Self) -> R,
@@ -820,24 +843,16 @@ where
     }
 }
 
-impl<'a, 'b, 'c, 'tcx, E> SpecializedEncoder<&'b [(ty::Predicate<'c>, Span)]>
-    for CacheEncoder<'a, 'tcx, E>
+impl<'a, 'b, 'tcx, E> SpecializedEncoder<ty::Predicate<'b>> for CacheEncoder<'a, 'tcx, E>
 where
     E: 'a + TyEncoder,
 {
     #[inline]
-    fn specialized_encode(
-        &mut self,
-        predicates: &&'b [(ty::Predicate<'c>, Span)],
-    ) -> Result<(), Self::Error> {
-        debug_assert!(self.tcx.lift(*predicates).is_some());
-        let predicates = unsafe {
-            std::mem::transmute::<
-                &&'b [(ty::Predicate<'c>, Span)],
-                &&'tcx [(ty::Predicate<'tcx>, Span)],
-            >(predicates)
-        };
-        ty_codec::encode_spanned_predicates(self, predicates, |encoder| {
+    fn specialized_encode(&mut self, predicate: &ty::Predicate<'b>) -> Result<(), Self::Error> {
+        debug_assert!(self.tcx.lift(predicate).is_some());
+        let predicate =
+            unsafe { std::mem::transmute::<&ty::Predicate<'b>, &ty::Predicate<'tcx>>(predicate) };
+        ty_codec::encode_with_shorthand(self, predicate, |encoder| {
             &mut encoder.predicate_shorthands
         })
     }
diff --git a/src/librustc_middle/ty/structural_impls.rs b/src/librustc_middle/ty/structural_impls.rs
index f04d31601ea..f736037b5c1 100644
--- a/src/librustc_middle/ty/structural_impls.rs
+++ b/src/librustc_middle/ty/structural_impls.rs
@@ -987,12 +987,34 @@ impl<'tcx> TypeFoldable<'tcx> for ty::Region<'tcx> {
 
 impl<'tcx> TypeFoldable<'tcx> for ty::Predicate<'tcx> {
     fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        let new = ty::PredicateKind::super_fold_with(self.kind, folder);
-        if new != *self.kind { folder.tcx().mk_predicate(new) } else { *self }
+        let new = ty::PredicateKind::super_fold_with(&self.inner.kind, folder);
+        if new != self.inner.kind { folder.tcx().mk_predicate(new) } else { *self }
     }
 
     fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
-        ty::PredicateKind::super_visit_with(self.kind, visitor)
+        ty::PredicateKind::super_visit_with(&self.inner.kind, visitor)
+    }
+
+    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
+        visitor.visit_predicate(*self)
+    }
+
+    fn has_vars_bound_at_or_above(&self, binder: ty::DebruijnIndex) -> bool {
+        self.inner.outer_exclusive_binder > binder
+    }
+
+    fn has_type_flags(&self, flags: ty::TypeFlags) -> bool {
+        self.inner.flags.intersects(flags)
+    }
+}
+
+pub(super) trait PredicateVisitor<'tcx>: TypeVisitor<'tcx> {
+    fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool;
+}
+
+impl<T: TypeVisitor<'tcx>> PredicateVisitor<'tcx> for T {
+    default fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool {
+        predicate.super_visit_with(self)
     }
 }
 
diff --git a/src/librustc_trait_selection/traits/fulfill.rs b/src/librustc_trait_selection/traits/fulfill.rs
index 106753ed809..32ab63458e7 100644
--- a/src/librustc_trait_selection/traits/fulfill.rs
+++ b/src/librustc_trait_selection/traits/fulfill.rs
@@ -321,7 +321,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> {
             ty::PredicateKind::Trait(ref data, _) => {
                 let trait_obligation = obligation.with(*data);
 
-                if data.is_global() {
+                if obligation.predicate.is_global() {
                     // no type variables present, can use evaluation for better caching.
                     // FIXME: consider caching errors too.
                     if infcx.predicate_must_hold_considering_regions(&obligation) {