Auto merge of #81177 - Aaron1011:fix/force-capture-tokens, r=petrochenkov

Force token collection to run when parsing nonterminals

Fixes #81007

Previously, we would fail to collect tokens in the proper place when
only builtin attributes were present. As a result, we would end up with
attribute tokens in the collected `TokenStream`, leading to duplication
when we attempted to prepend the attributes from the AST node.

We now explicitly track when token collection must be performed due to
nomterminal parsing.

2 files changed, 130 insertions, 0 deletions

diff --git a/src/test/ui/proc-macro/issue-81007-item-attrs.rs b/src/test/ui/proc-macro/issue-81007-item-attrs.rs
new file mode 100644
index 00000000000..ea27d54ee41
--- /dev/null
+++ b/src/test/ui/proc-macro/issue-81007-item-attrs.rs
@@ -0,0 +1,31 @@
+// check-pass
+// edition:2018
+// compile-flags: -Z span-debug
+// aux-build:test-macros.rs
+
+#![feature(rustc_attrs)]
+
+#![no_std] // Don't load unnecessary hygiene information from std
+extern crate std;
+
+#[macro_use] extern crate test_macros;
+
+macro_rules! capture_item {
+    ($item:item) => {
+        #[print_attr]
+        $item
+    }
+}
+
+capture_item! {
+    /// A doc comment
+    struct Foo {}
+}
+
+capture_item! {
+    #[rustc_dummy]
+    /// Another comment comment
+    struct Bar {}
+}
+
+fn main() {}
diff --git a/src/test/ui/proc-macro/issue-81007-item-attrs.stdout b/src/test/ui/proc-macro/issue-81007-item-attrs.stdout
new file mode 100644
index 00000000000..6f880a12021
--- /dev/null
+++ b/src/test/ui/proc-macro/issue-81007-item-attrs.stdout
@@ -0,0 +1,99 @@
+PRINT-ATTR INPUT (DISPLAY): #[doc = r" A doc comment"] struct Foo { }
+PRINT-ATTR INPUT (DEBUG): TokenStream [
+    Punct {
+        ch: '#',
+        spacing: Alone,
+        span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0),
+    },
+    Group {
+        delimiter: Bracket,
+        stream: TokenStream [
+            Ident {
+                ident: "doc",
+                span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0),
+            },
+            Punct {
+                ch: '=',
+                spacing: Alone,
+                span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0),
+            },
+            Literal {
+                kind: StrRaw(0),
+                symbol: " A doc comment",
+                suffix: None,
+                span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0),
+            },
+        ],
+        span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0),
+    },
+    Ident {
+        ident: "struct",
+        span: $DIR/issue-81007-item-attrs.rs:22:5: 22:11 (#0),
+    },
+    Ident {
+        ident: "Foo",
+        span: $DIR/issue-81007-item-attrs.rs:22:12: 22:15 (#0),
+    },
+    Group {
+        delimiter: Brace,
+        stream: TokenStream [],
+        span: $DIR/issue-81007-item-attrs.rs:22:16: 22:18 (#0),
+    },
+]
+PRINT-ATTR INPUT (DISPLAY): #[rustc_dummy] #[doc = r" Another comment comment"] struct Bar { }
+PRINT-ATTR INPUT (DEBUG): TokenStream [
+    Punct {
+        ch: '#',
+        spacing: Alone,
+        span: $DIR/issue-81007-item-attrs.rs:26:5: 26:6 (#0),
+    },
+    Group {
+        delimiter: Bracket,
+        stream: TokenStream [
+            Ident {
+                ident: "rustc_dummy",
+                span: $DIR/issue-81007-item-attrs.rs:26:7: 26:18 (#0),
+            },
+        ],
+        span: $DIR/issue-81007-item-attrs.rs:26:6: 26:19 (#0),
+    },
+    Punct {
+        ch: '#',
+        spacing: Alone,
+        span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0),
+    },
+    Group {
+        delimiter: Bracket,
+        stream: TokenStream [
+            Ident {
+                ident: "doc",
+                span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0),
+            },
+            Punct {
+                ch: '=',
+                spacing: Alone,
+                span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0),
+            },
+            Literal {
+                kind: StrRaw(0),
+                symbol: " Another comment comment",
+                suffix: None,
+                span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0),
+            },
+        ],
+        span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0),
+    },
+    Ident {
+        ident: "struct",
+        span: $DIR/issue-81007-item-attrs.rs:28:5: 28:11 (#0),
+    },
+    Ident {
+        ident: "Bar",
+        span: $DIR/issue-81007-item-attrs.rs:28:12: 28:15 (#0),
+    },
+    Group {
+        delimiter: Brace,
+        stream: TokenStream [],
+        span: $DIR/issue-81007-item-attrs.rs:28:16: 28:18 (#0),
+    },
+]