Use global recursion limit when evaluating inhabitedness

5 files changed, 55 insertions, 47 deletions

diff --git a/src/librustc/ty/inhabitedness/mod.rs b/src/librustc/ty/inhabitedness/mod.rs
index 51d046728c2..24ca476e5ff 100644
--- a/src/librustc/ty/inhabitedness/mod.rs
+++ b/src/librustc/ty/inhabitedness/mod.rs
@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use util::nodemap::FxHashSet;
+use util::nodemap::{FxHashMap, FxHashSet};
 use ty::context::TyCtxt;
 use ty::{AdtDef, VariantDef, FieldDef, TyS};
 use ty::{DefId, Substs};
@@ -62,26 +62,17 @@ mod def_id_forest;
 // This code should only compile in modules where the uninhabitedness of Foo is
 // visible.
 
-const ARBITRARY_RECURSION_LIMIT: u32 = 24;
-
 impl<'a, 'gcx, 'tcx> AdtDef {
     /// Calculate the forest of DefIds from which this adt is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
-                recursion_depth: u32,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>,
                 substs: &'tcx Substs<'tcx>) -> DefIdForest
     {
-        if !visited.insert((self.did, substs)) {
-            return DefIdForest::empty();
-        }
-
-        let ret = DefIdForest::intersection(tcx, self.variants.iter().map(|v| {
-            v.uninhabited_from(visited, recursion_depth, tcx, substs, self.adt_kind())
-        }));
-        visited.remove(&(self.did, substs));
-        ret
+        DefIdForest::intersection(tcx, self.variants.iter().map(|v| {
+            v.uninhabited_from(visited, tcx, substs, self.adt_kind())
+        }))
     }
 }
 
@@ -89,8 +80,7 @@ impl<'a, 'gcx, 'tcx> VariantDef {
     /// Calculate the forest of DefIds from which this variant is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
-                recursion_depth: u32,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>,
                 substs: &'tcx Substs<'tcx>,
                 adt_kind: AdtKind) -> DefIdForest
@@ -98,17 +88,17 @@ impl<'a, 'gcx, 'tcx> VariantDef {
         match adt_kind {
             AdtKind::Union => {
                 DefIdForest::intersection(tcx, self.fields.iter().map(|f| {
-                    f.uninhabited_from(visited, recursion_depth, tcx, substs, false)
+                    f.uninhabited_from(visited, tcx, substs, false)
                 }))
             },
             AdtKind::Struct => {
                 DefIdForest::union(tcx, self.fields.iter().map(|f| {
-                    f.uninhabited_from(visited, recursion_depth, tcx, substs, false)
+                    f.uninhabited_from(visited, tcx, substs, false)
                 }))
             },
             AdtKind::Enum => {
                 DefIdForest::union(tcx, self.fields.iter().map(|f| {
-                    f.uninhabited_from(visited, recursion_depth, tcx, substs, true)
+                    f.uninhabited_from(visited, tcx, substs, true)
                 }))
             },
         }
@@ -119,14 +109,13 @@ impl<'a, 'gcx, 'tcx> FieldDef {
     /// Calculate the forest of DefIds from which this field is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
-                recursion_depth: u32,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>,
                 substs: &'tcx Substs<'tcx>,
                 is_enum: bool) -> DefIdForest
     {
         let mut data_uninhabitedness = move || {
-            self.ty(tcx, substs).uninhabited_from(visited, recursion_depth, tcx)
+            self.ty(tcx, substs).uninhabited_from(visited, tcx)
         };
         // FIXME(canndrew): Currently enum fields are (incorrectly) stored with
         // Visibility::Invisible so we need to override self.vis if we're
@@ -151,15 +140,9 @@ impl<'a, 'gcx, 'tcx> TyS<'tcx> {
     /// Calculate the forest of DefIds from which this type is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
-                mut recursion_depth: u32,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>) -> DefIdForest
     {
-        recursion_depth += 1;
-        if recursion_depth >= ARBITRARY_RECURSION_LIMIT {
-            return DefIdForest::empty();
-        }
-
         match tcx.lift_to_global(&self) {
             Some(global_ty) => {
                 {
@@ -168,13 +151,13 @@ impl<'a, 'gcx, 'tcx> TyS<'tcx> {
                         return forest.clone();
                     }
                 }
-                let forest = global_ty.uninhabited_from_inner(visited, recursion_depth, tcx);
+                let forest = global_ty.uninhabited_from_inner(visited, tcx);
                 let mut cache = tcx.inhabitedness_cache.borrow_mut();
                 cache.insert(global_ty, forest.clone());
                 forest
             },
             None => {
-                let forest = self.uninhabited_from_inner(visited, recursion_depth, tcx);
+                let forest = self.uninhabited_from_inner(visited, tcx);
                 forest
             },
         }
@@ -182,30 +165,54 @@ impl<'a, 'gcx, 'tcx> TyS<'tcx> {
 
     fn uninhabited_from_inner(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
-                recursion_depth: u32,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>) -> DefIdForest
     {
         match self.sty {
             TyAdt(def, substs) => {
-                def.uninhabited_from(visited, recursion_depth, tcx, substs)
+                {
+                    let mut substs_set = visited.entry(def.did).or_insert(FxHashSet::default());
+                    if !substs_set.insert(substs) {
+                        // We are already calculating the inhabitedness of this type.
+                        // The type must contain a reference to itself. Break the
+                        // infinite loop.
+                        return DefIdForest::empty();
+                    }
+                    if substs_set.len() >= tcx.sess.recursion_limit.get() / 4 {
+                        // We have gone very deep, reinstantiating this ADT inside
+                        // itself with different type arguments. We are probably
+                        // hitting an infinite loop. For example, it's possible to write:
+                        //                a type Foo<T>
+                        //      which contains a Foo<(T, T)>
+                        //      which contains a Foo<((T, T), (T, T))>
+                        //      which contains a Foo<(((T, T), (T, T)), ((T, T), (T, T)))>
+                        //      etc.
+                        let error = format!("reached recursion limit while checking
+                                             inhabitedness of `{}`", self);
+                        tcx.sess.fatal(&error);
+                    }
+                }
+                let ret = def.uninhabited_from(visited, tcx, substs);
+                let mut substs_set = visited.get_mut(&def.did).unwrap();
+                substs_set.remove(substs);
+                ret
             },
 
             TyNever => DefIdForest::full(tcx),
             TyTuple(ref tys, _) => {
                 DefIdForest::union(tcx, tys.iter().map(|ty| {
-                    ty.uninhabited_from(visited, recursion_depth, tcx)
+                    ty.uninhabited_from(visited, tcx)
                 }))
             },
             TyArray(ty, len) => {
                 if len == 0 {
                     DefIdForest::empty()
                 } else {
-                    ty.uninhabited_from(visited, recursion_depth, tcx)
+                    ty.uninhabited_from(visited, tcx)
                 }
             }
             TyRef(_, ref tm) => {
-                tm.ty.uninhabited_from(visited, recursion_depth, tcx)
+                tm.ty.uninhabited_from(visited, tcx)
             }
 
             _ => DefIdForest::empty(),
diff --git a/src/librustc/ty/sty.rs b/src/librustc/ty/sty.rs
index 61b77492426..862bc15c052 100644
--- a/src/librustc/ty/sty.rs
+++ b/src/librustc/ty/sty.rs
@@ -24,7 +24,7 @@ use std::cmp::Ordering;
 use syntax::abi;
 use syntax::ast::{self, Name};
 use syntax::symbol::{keywords, InternedString};
-use util::nodemap::FxHashSet;
+use util::nodemap::FxHashMap;
 
 use serialize;
 
@@ -1018,8 +1018,8 @@ impl<'a, 'gcx, 'tcx> TyS<'tcx> {
     /// This code should only compile in modules where the uninhabitedness of Foo is
     /// visible.
     pub fn is_uninhabited_from(&self, module: DefId, tcx: TyCtxt<'a, 'gcx, 'tcx>) -> bool {
-        let mut visited = FxHashSet::default();
-        let forest = self.uninhabited_from(&mut visited, 0, tcx);
+        let mut visited = FxHashMap::default();
+        let forest = self.uninhabited_from(&mut visited, tcx);
 
         // To check whether this type is uninhabited at all (not just from the
         // given node) you could check whether the forest is empty.
diff --git a/src/librustc_const_eval/_match.rs b/src/librustc_const_eval/_match.rs
index 101c43332a3..78c4027aa43 100644
--- a/src/librustc_const_eval/_match.rs
+++ b/src/librustc_const_eval/_match.rs
@@ -17,7 +17,7 @@ use eval::{compare_const_vals};
 
 use rustc_const_math::ConstInt;
 
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::indexed_vec::Idx;
 
 use pattern::{FieldPattern, Pattern, PatternKind};
@@ -404,8 +404,8 @@ fn all_constructors<'a, 'tcx: 'a>(cx: &mut MatchCheckCtxt<'a, 'tcx>,
         }
         ty::TyAdt(def, substs) if def.is_enum() && def.variants.len() != 1 => {
             def.variants.iter().filter_map(|v| {
-                let mut visited = FxHashSet::default();
-                let forest = v.uninhabited_from(&mut visited, 0,
+                let mut visited = FxHashMap::default();
+                let forest = v.uninhabited_from(&mut visited,
                                                 cx.tcx, substs,
                                                 AdtKind::Enum);
                 if forest.contains(cx.tcx, cx.module)
diff --git a/src/librustc_mir/build/matches/simplify.rs b/src/librustc_mir/build/matches/simplify.rs
index f6bfed8c7ab..efddee2c933 100644
--- a/src/librustc_mir/build/matches/simplify.rs
+++ b/src/librustc_mir/build/matches/simplify.rs
@@ -26,7 +26,7 @@ use build::{BlockAnd, BlockAndExtension, Builder};
 use build::matches::{Binding, MatchPair, Candidate};
 use hair::*;
 use rustc::mir::*;
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxHashMap;
 
 use std::mem;
 
@@ -102,8 +102,8 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
                 if self.hir.tcx().sess.features.borrow().never_type {
                     let irrefutable = adt_def.variants.iter().enumerate().all(|(i, v)| {
                         i == variant_index || {
-                            let mut visited = FxHashSet::default();
-                            let node_set = v.uninhabited_from(&mut visited, 0,
+                            let mut visited = FxHashMap::default();
+                            let node_set = v.uninhabited_from(&mut visited,
                                                               self.hir.tcx(),
                                                               substs,
                                                               adt_def.adt_kind());
diff --git a/src/test/compile-fail/inhabitedness-infinite-loop.rs b/src/test/compile-fail/inhabitedness-infinite-loop.rs
index eba0da3a216..91b85d7510a 100644
--- a/src/test/compile-fail/inhabitedness-infinite-loop.rs
+++ b/src/test/compile-fail/inhabitedness-infinite-loop.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// error-pattern:reached recursion limit
+
 #![feature(never_type)]
 
 struct Foo<'a, T: 'a> {
@@ -17,7 +19,6 @@ struct Foo<'a, T: 'a> {
 
 fn wub(f: Foo<!>) {
     match f {}
-    //~^ ERROR non-exhaustive
 }
 
 fn main() {}