rustdoc: catch and don't blow up on impl Trait cycles

An odd feature of Rust is that `Foo` is invalid, but `Bar` is okay:

    type Foo<'a, 'b> = Box<dyn PartialEq<Foo<'a, 'b>>>;
    type Bar<'a, 'b> = impl PartialEq<Bar<'a, 'b>>;

To get it right, track every time rustdoc descends into a type alias,
so if it shows up twice, it can be write the path instead of
infinitely expanding it.

6 files changed, 127 insertions, 19 deletions

diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs
index 8ccdb16b784..1531e7fc7b9 100644
--- a/src/librustdoc/clean/mod.rs
+++ b/src/librustdoc/clean/mod.rs
@@ -1529,7 +1529,9 @@ fn maybe_expand_private_type_alias<'tcx>(
     let Res::Def(DefKind::TyAlias, def_id) = path.res else { return None };
     // Substitute private type aliases
     let def_id = def_id.as_local()?;
-    let alias = if !cx.cache.effective_visibilities.is_exported(cx.tcx, def_id.to_def_id()) {
+    let alias = if !cx.cache.effective_visibilities.is_exported(cx.tcx, def_id.to_def_id())
+        && !cx.current_type_aliases.contains_key(&def_id.to_def_id())
+    {
         &cx.tcx.hir().expect_item(def_id).kind
     } else {
         return None;
@@ -1609,7 +1611,7 @@ fn maybe_expand_private_type_alias<'tcx>(
         }
     }
 
-    Some(cx.enter_alias(substs, |cx| clean_ty(ty, cx)))
+    Some(cx.enter_alias(substs, def_id.to_def_id(), |cx| clean_ty(ty, cx)))
 }
 
 pub(crate) fn clean_ty<'tcx>(ty: &hir::Ty<'tcx>, cx: &mut DocContext<'tcx>) -> Type {
@@ -1700,7 +1702,7 @@ fn normalize<'tcx>(
 pub(crate) fn clean_middle_ty<'tcx>(
     bound_ty: ty::Binder<'tcx, Ty<'tcx>>,
     cx: &mut DocContext<'tcx>,
-    def_id: Option<DefId>,
+    parent_def_id: Option<DefId>,
 ) -> Type {
     let bound_ty = normalize(cx, bound_ty).unwrap_or(bound_ty);
     match *bound_ty.skip_binder().kind() {
@@ -1830,7 +1832,9 @@ pub(crate) fn clean_middle_ty<'tcx>(
             Tuple(t.iter().map(|t| clean_middle_ty(bound_ty.rebind(t), cx, None)).collect())
         }
 
-        ty::Alias(ty::Projection, ref data) => clean_projection(bound_ty.rebind(*data), cx, def_id),
+        ty::Alias(ty::Projection, ref data) => {
+            clean_projection(bound_ty.rebind(*data), cx, parent_def_id)
+        }
 
         ty::Param(ref p) => {
             if let Some(bounds) = cx.impl_trait_bounds.remove(&p.index.into()) {
@@ -1841,15 +1845,30 @@ pub(crate) fn clean_middle_ty<'tcx>(
         }
 
         ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) => {
-            // Grab the "TraitA + TraitB" from `impl TraitA + TraitB`,
-            // by looking up the bounds associated with the def_id.
-            let bounds = cx
-                .tcx
-                .explicit_item_bounds(def_id)
-                .subst_iter_copied(cx.tcx, substs)
-                .map(|(bound, _)| bound)
-                .collect::<Vec<_>>();
-            clean_middle_opaque_bounds(cx, bounds)
+            // If it's already in the same alias, don't get an infinite loop.
+            if cx.current_type_aliases.contains_key(&def_id) {
+                let path =
+                    external_path(cx, def_id, false, ThinVec::new(), bound_ty.rebind(substs));
+                Type::Path { path }
+            } else {
+                *cx.current_type_aliases.entry(def_id).or_insert(0) += 1;
+                // Grab the "TraitA + TraitB" from `impl TraitA + TraitB`,
+                // by looking up the bounds associated with the def_id.
+                let bounds = cx
+                    .tcx
+                    .explicit_item_bounds(def_id)
+                    .subst_iter_copied(cx.tcx, substs)
+                    .map(|(bound, _)| bound)
+                    .collect::<Vec<_>>();
+                let ty = clean_middle_opaque_bounds(cx, bounds);
+                if let Some(count) = cx.current_type_aliases.get_mut(&def_id) {
+                    *count -= 1;
+                    if *count == 0 {
+                        cx.current_type_aliases.remove(&def_id);
+                    }
+                }
+                ty
+            }
         }
 
         ty::Closure(..) => panic!("Closure"),
@@ -2229,13 +2248,17 @@ fn clean_maybe_renamed_item<'tcx>(
                 generics: clean_generics(ty.generics, cx),
             }),
             ItemKind::TyAlias(hir_ty, generics) => {
+                *cx.current_type_aliases.entry(def_id).or_insert(0) += 1;
                 let rustdoc_ty = clean_ty(hir_ty, cx);
                 let ty = clean_middle_ty(ty::Binder::dummy(hir_ty_to_ty(cx.tcx, hir_ty)), cx, None);
-                TypedefItem(Box::new(Typedef {
-                    type_: rustdoc_ty,
-                    generics: clean_generics(generics, cx),
-                    item_type: Some(ty),
-                }))
+                let generics = clean_generics(generics, cx);
+                if let Some(count) = cx.current_type_aliases.get_mut(&def_id) {
+                    *count -= 1;
+                    if *count == 0 {
+                        cx.current_type_aliases.remove(&def_id);
+                    }
+                }
+                TypedefItem(Box::new(Typedef { type_: rustdoc_ty, generics, item_type: Some(ty) }))
             }
             ItemKind::Enum(ref def, generics) => EnumItem(Enum {
                 variants: def.variants.iter().map(|v| clean_variant(v, cx)).collect(),
diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs
index 46834a1d7f4..3a0c2ab0297 100644
--- a/src/librustdoc/core.rs
+++ b/src/librustdoc/core.rs
@@ -46,6 +46,7 @@ pub(crate) struct DocContext<'tcx> {
     // for expanding type aliases at the HIR level:
     /// Table `DefId` of type, lifetime, or const parameter -> substituted type, lifetime, or const
     pub(crate) substs: DefIdMap<clean::SubstParam>,
+    pub(crate) current_type_aliases: DefIdMap<usize>,
     /// Table synthetic type parameter for `impl Trait` in argument position -> bounds
     pub(crate) impl_trait_bounds: FxHashMap<ImplTraitParam, Vec<clean::GenericBound>>,
     /// Auto-trait or blanket impls processed so far, as `(self_ty, trait_def_id)`.
@@ -82,13 +83,25 @@ impl<'tcx> DocContext<'tcx> {
 
     /// Call the closure with the given parameters set as
     /// the substitutions for a type alias' RHS.
-    pub(crate) fn enter_alias<F, R>(&mut self, substs: DefIdMap<clean::SubstParam>, f: F) -> R
+    pub(crate) fn enter_alias<F, R>(
+        &mut self,
+        substs: DefIdMap<clean::SubstParam>,
+        def_id: DefId,
+        f: F,
+    ) -> R
     where
         F: FnOnce(&mut Self) -> R,
     {
         let old_substs = mem::replace(&mut self.substs, substs);
+        *self.current_type_aliases.entry(def_id).or_insert(0) += 1;
         let r = f(self);
         self.substs = old_substs;
+        if let Some(count) = self.current_type_aliases.get_mut(&def_id) {
+            *count -= 1;
+            if *count == 0 {
+                self.current_type_aliases.remove(&def_id);
+            }
+        }
         r
     }
 
@@ -327,6 +340,7 @@ pub(crate) fn run_global_ctxt(
         external_traits: Default::default(),
         active_extern_traits: Default::default(),
         substs: Default::default(),
+        current_type_aliases: Default::default(),
         impl_trait_bounds: Default::default(),
         generated_synthetics: Default::default(),
         auto_traits,
diff --git a/tests/rustdoc-ui/issue-110629-private-type-cycle-dyn.rs b/tests/rustdoc-ui/issue-110629-private-type-cycle-dyn.rs
new file mode 100644
index 00000000000..c920a815fda
--- /dev/null
+++ b/tests/rustdoc-ui/issue-110629-private-type-cycle-dyn.rs
@@ -0,0 +1,12 @@
+type Bar<'a, 'b> = Box<dyn PartialEq<Bar<'a, 'b>>>;
+//~^ ERROR cycle detected when expanding type alias
+
+fn bar<'a, 'b>(i: &'a i32) -> Bar<'a, 'b> {
+    Box::new(i)
+}
+
+fn main() {
+    let meh = 42;
+    let muh = 42;
+    assert!(bar(&meh) == bar(&muh));
+}
diff --git a/tests/rustdoc-ui/issue-110629-private-type-cycle-dyn.stderr b/tests/rustdoc-ui/issue-110629-private-type-cycle-dyn.stderr
new file mode 100644
index 00000000000..79e1b753112
--- /dev/null
+++ b/tests/rustdoc-ui/issue-110629-private-type-cycle-dyn.stderr
@@ -0,0 +1,25 @@
+error[E0391]: cycle detected when expanding type alias `Bar`
+  --> $DIR/issue-110629-private-type-cycle-dyn.rs:1:38
+   |
+LL | type Bar<'a, 'b> = Box<dyn PartialEq<Bar<'a, 'b>>>;
+   |                                      ^^^^^^^^^^^
+   |
+   = note: ...which immediately requires expanding type alias `Bar` again
+   = note: type aliases cannot be recursive
+   = help: consider using a struct, enum, or union instead to break the cycle
+   = help: see <https://doc.rust-lang.org/reference/types.html#recursive-types> for more information
+note: cycle used when collecting item types in top-level module
+  --> $DIR/issue-110629-private-type-cycle-dyn.rs:1:1
+   |
+LL | / type Bar<'a, 'b> = Box<dyn PartialEq<Bar<'a, 'b>>>;
+LL | |
+LL | |
+LL | | fn bar<'a, 'b>(i: &'a i32) -> Bar<'a, 'b> {
+...  |
+LL | |     assert!(bar(&meh) == bar(&muh));
+LL | | }
+   | |_^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0391`.
diff --git a/tests/rustdoc-ui/issue-110629-private-type-cycle.rs b/tests/rustdoc-ui/issue-110629-private-type-cycle.rs
new file mode 100644
index 00000000000..2d46ddbfa6e
--- /dev/null
+++ b/tests/rustdoc-ui/issue-110629-private-type-cycle.rs
@@ -0,0 +1,15 @@
+// check-pass
+
+#![feature(type_alias_impl_trait)]
+
+type Bar<'a, 'b> = impl PartialEq<Bar<'a, 'b>> + std::fmt::Debug;
+
+fn bar<'a, 'b>(i: &'a i32) -> Bar<'a, 'b> {
+    i
+}
+
+fn main() {
+    let meh = 42;
+    let muh = 42;
+    assert_eq!(bar(&meh), bar(&muh));
+}
diff --git a/tests/rustdoc/issue-110629-private-type-cycle.rs b/tests/rustdoc/issue-110629-private-type-cycle.rs
new file mode 100644
index 00000000000..a4efbb098f7
--- /dev/null
+++ b/tests/rustdoc/issue-110629-private-type-cycle.rs
@@ -0,0 +1,19 @@
+// compile-flags: --document-private-items
+
+#![feature(type_alias_impl_trait)]
+
+type Bar<'a, 'b> = impl PartialEq<Bar<'a, 'b>> + std::fmt::Debug;
+
+// @has issue_110629_private_type_cycle/type.Bar.html
+// @has - '//pre[@class="rust item-decl"]' \
+//     "pub(crate) type Bar<'a, 'b> = impl PartialEq<Bar<'a, 'b>> + Debug;"
+
+fn bar<'a, 'b>(i: &'a i32) -> Bar<'a, 'b> {
+    i
+}
+
+fn main() {
+    let meh = 42;
+    let muh = 42;
+    assert_eq!(bar(&meh), bar(&muh));
+}