Rollup merge of #138599 - adwinwhite:recursive-overflow, r=wesleywiser

avoid overflow when generating debuginfo for expanding recursive types

Fixes #135093
Fixes #121538
Fixes #107362
Fixes #100618
Fixes #115994

The overflow happens because expanding recursive types keep creating new nested types when recurring into sub fields.
I fixed that by returning an empty stub node when expanding recursion is detected.

2 files changed, 50 insertions, 0 deletions

diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
index ae2ab32ef53..56fb12d3c22 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
@@ -247,6 +247,16 @@ pub(super) fn stub<'ll, 'tcx>(
     StubInfo { metadata, unique_type_id }
 }
 
+struct AdtStackPopGuard<'ll, 'tcx, 'a> {
+    cx: &'a CodegenCx<'ll, 'tcx>,
+}
+
+impl<'ll, 'tcx, 'a> Drop for AdtStackPopGuard<'ll, 'tcx, 'a> {
+    fn drop(&mut self) {
+        debug_context(self.cx).adt_stack.borrow_mut().pop();
+    }
+}
+
 /// This function enables creating debuginfo nodes that can recursively refer to themselves.
 /// It will first insert the given stub into the type map and only then execute the `members`
 /// and `generics` closures passed in. These closures have access to the stub so they can
@@ -261,6 +271,44 @@ pub(super) fn build_type_with_children<'ll, 'tcx>(
 ) -> DINodeCreationResult<'ll> {
     assert_eq!(debug_context(cx).type_map.di_node_for_unique_id(stub_info.unique_type_id), None);
 
+    let mut _adt_stack_pop_guard = None;
+    if let UniqueTypeId::Ty(ty, ..) = stub_info.unique_type_id
+        && let ty::Adt(adt_def, args) = ty.kind()
+    {
+        let def_id = adt_def.did();
+        // If any sub type reference the original type definition and the sub type has a type
+        // parameter that strictly contains the original parameter, the original type is a recursive
+        // type that can expanding indefinitely. Example,
+        // ```
+        // enum Recursive<T> {
+        //     Recurse(*const Recursive<Wrap<T>>),
+        //     Item(T),
+        // }
+        // ```
+        let is_expanding_recursive =
+            debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| {
+                if def_id == *parent_def_id {
+                    args.iter().zip(parent_args.iter()).any(|(arg, parent_arg)| {
+                        if let (Some(arg), Some(parent_arg)) = (arg.as_type(), parent_arg.as_type())
+                        {
+                            arg != parent_arg && arg.contains(parent_arg)
+                        } else {
+                            false
+                        }
+                    })
+                } else {
+                    false
+                }
+            });
+        if is_expanding_recursive {
+            // FIXME: indicate that this is an expanding recursive type in stub metadata?
+            return DINodeCreationResult::new(stub_info.metadata, false);
+        } else {
+            debug_context(cx).adt_stack.borrow_mut().push((def_id, args));
+            _adt_stack_pop_guard = Some(AdtStackPopGuard { cx });
+        }
+    }
+
     debug_context(cx).type_map.insert(stub_info.unique_type_id, stub_info.metadata);
 
     let members: SmallVec<_> =
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index 0f94a1dbb0d..c5085927923 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -66,6 +66,7 @@ pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> {
     created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>,
 
     type_map: metadata::TypeMap<'ll, 'tcx>,
+    adt_stack: RefCell<Vec<(DefId, GenericArgsRef<'tcx>)>>,
     namespace_map: RefCell<DefIdMap<&'ll DIScope>>,
     recursion_marker_type: OnceCell<&'ll DIType>,
 }
@@ -80,6 +81,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> {
             builder,
             created_files: Default::default(),
             type_map: Default::default(),
+            adt_stack: Default::default(),
             namespace_map: RefCell::new(Default::default()),
             recursion_marker_type: OnceCell::new(),
         }