Capture tokens for Pat used in macro_rules! argument

This extends PR #73293 to handle patterns (Pat). Unlike expressions,
patterns do not support custom attributes, so we only need to capture
tokens during macro_rules! argument parsing.

13 files changed, 85 insertions, 10 deletions

diff --git a/src/librustc_ast/ast.rs b/src/librustc_ast/ast.rs
index 6dff02486ff..e06881711c4 100644
--- a/src/librustc_ast/ast.rs
+++ b/src/librustc_ast/ast.rs
@@ -550,6 +550,7 @@ pub struct Pat {
     pub id: NodeId,
     pub kind: PatKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 impl Pat {
@@ -2138,6 +2139,7 @@ impl Param {
                 id: DUMMY_NODE_ID,
                 kind: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None),
                 span,
+                tokens: None,
             }),
             span,
             ty,
diff --git a/src/librustc_ast/mut_visit.rs b/src/librustc_ast/mut_visit.rs
index df6e8218f6c..1e922d3415f 100644
--- a/src/librustc_ast/mut_visit.rs
+++ b/src/librustc_ast/mut_visit.rs
@@ -1053,7 +1053,7 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(
 }
 
 pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
-    let Pat { id, kind, span } = pat.deref_mut();
+    let Pat { id, kind, span, tokens: _ } = pat.deref_mut();
     vis.visit_id(id);
     match kind {
         PatKind::Wild | PatKind::Rest => {}
diff --git a/src/librustc_expand/base.rs b/src/librustc_expand/base.rs
index 0a5e1f76d10..4c01cb8159a 100644
--- a/src/librustc_expand/base.rs
+++ b/src/librustc_expand/base.rs
@@ -551,6 +551,7 @@ impl MacResult for MacEager {
                     id: ast::DUMMY_NODE_ID,
                     span: e.span,
                     kind: PatKind::Lit(e),
+                    tokens: None,
                 }));
             }
         }
@@ -597,7 +598,7 @@ impl DummyResult {
 
     /// A plain dummy pattern.
     pub fn raw_pat(sp: Span) -> ast::Pat {
-        ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp }
+        ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None }
     }
 
     /// A plain dummy type.
diff --git a/src/librustc_expand/build.rs b/src/librustc_expand/build.rs
index 7f4737a38ab..9490b62aa17 100644
--- a/src/librustc_expand/build.rs
+++ b/src/librustc_expand/build.rs
@@ -392,7 +392,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> {
-        P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span })
+        P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span, tokens: None })
     }
     pub fn pat_wild(&self, span: Span) -> P<ast::Pat> {
         self.pat(span, PatKind::Wild)
diff --git a/src/librustc_expand/placeholders.rs b/src/librustc_expand/placeholders.rs
index 0788823b0e7..29fb4c95ec6 100644
--- a/src/librustc_expand/placeholders.rs
+++ b/src/librustc_expand/placeholders.rs
@@ -38,7 +38,8 @@ pub fn placeholder(
         })
     };
     let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span });
-    let pat = || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span });
+    let pat =
+        || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span, tokens: None });
 
     match kind {
         AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()),
@@ -85,6 +86,7 @@ pub fn placeholder(
             id,
             span,
             kind: ast::PatKind::MacCall(mac_placeholder()),
+            tokens: None,
         })),
         AstFragmentKind::Ty => {
             AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::MacCall(mac_placeholder()) }))
diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs
index 49deb3474d9..7b7235eed36 100644
--- a/src/librustc_parse/lib.rs
+++ b/src/librustc_parse/lib.rs
@@ -263,6 +263,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
         Nonterminal::NtItem(ref item) => {
             prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
         }
+        Nonterminal::NtPat(ref pat) => pat.tokens.clone(),
         Nonterminal::NtIdent(ident, is_raw) => {
             Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
         }
diff --git a/src/librustc_parse/parser/diagnostics.rs b/src/librustc_parse/parser/diagnostics.rs
index be01c7f8498..12efe391fb9 100644
--- a/src/librustc_parse/parser/diagnostics.rs
+++ b/src/librustc_parse/parser/diagnostics.rs
@@ -26,6 +26,7 @@ pub(super) fn dummy_arg(ident: Ident) -> Param {
         id: ast::DUMMY_NODE_ID,
         kind: PatKind::Ident(BindingMode::ByValue(Mutability::Not), ident, None),
         span: ident.span,
+        tokens: None,
     });
     let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID };
     Param {
@@ -83,7 +84,12 @@ impl RecoverQPath for Pat {
         self.to_ty()
     }
     fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self {
-        Self { span: path.span, kind: PatKind::Path(qself, path), id: ast::DUMMY_NODE_ID }
+        Self {
+            span: path.span,
+            kind: PatKind::Path(qself, path),
+            id: ast::DUMMY_NODE_ID,
+            tokens: None,
+        }
     }
 }
 
@@ -1526,7 +1532,8 @@ impl<'a> Parser<'a> {
         .emit();
 
         // Pretend the pattern is `_`, to avoid duplicate errors from AST validation.
-        let pat = P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID });
+        let pat =
+            P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID, tokens: None });
         Ok((pat, ty))
     }
 
diff --git a/src/librustc_parse/parser/nonterminal.rs b/src/librustc_parse/parser/nonterminal.rs
index 12139771bbf..f40cd1131d2 100644
--- a/src/librustc_parse/parser/nonterminal.rs
+++ b/src/librustc_parse/parser/nonterminal.rs
@@ -116,7 +116,14 @@ impl<'a> Parser<'a> {
                 Some(s) => token::NtStmt(s),
                 None => return Err(self.struct_span_err(self.token.span, "expected a statement")),
             },
-            NonterminalKind::Pat => token::NtPat(self.parse_pat(None)?),
+            NonterminalKind::Pat => {
+                let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
+                // We have have eaten an NtPat, which could already have tokens
+                if pat.tokens.is_none() {
+                    pat.tokens = Some(tokens);
+                }
+                token::NtPat(pat)
+            }
             NonterminalKind::Expr => {
                 let (mut expr, tokens) = self.collect_tokens(|this| this.parse_expr())?;
                 // If we captured tokens during parsing (due to outer attributes),
diff --git a/src/librustc_parse/parser/pat.rs b/src/librustc_parse/parser/pat.rs
index 1d8e6cc9bac..2c0133a24dc 100644
--- a/src/librustc_parse/parser/pat.rs
+++ b/src/librustc_parse/parser/pat.rs
@@ -1007,6 +1007,6 @@ impl<'a> Parser<'a> {
     }
 
     fn mk_pat(&self, span: Span, kind: PatKind) -> P<Pat> {
-        P(Pat { kind, span, id: ast::DUMMY_NODE_ID })
+        P(Pat { kind, span, id: ast::DUMMY_NODE_ID, tokens: None })
     }
 }
diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
index 8d4101b6526..633d153e391 100644
--- a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -164,6 +164,7 @@ fn iter_exprs(depth: usize, f: &mut dyn FnMut(P<Expr>)) {
                     id: DUMMY_NODE_ID,
                     kind: PatKind::Wild,
                     span: DUMMY_SP,
+                    tokens: None,
                 });
                 iter_exprs(depth - 1, &mut |e| g(ExprKind::Let(pat.clone(), e)))
             },
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
index a404ddace9b..2ff6ad6d68f 100644
--- a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
+++ b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
@@ -1,12 +1,20 @@
 // aux-build:test-macros.rs
 // check-pass
+// compile-flags: -Z span-debug
+// normalize-stdout-test "#\d+" -> "#CTXT"
 
 extern crate test_macros;
-use test_macros::recollect;
+use test_macros::print_bang;
 
 macro_rules! use_expr {
     ($expr:expr) => {
-        recollect!($expr)
+        print_bang!($expr)
+    }
+}
+
+macro_rules! use_pat {
+    ($pat:pat) => {
+        print_bang!($pat)
     }
 }
 
@@ -17,6 +25,10 @@ impl Foo {
     fn use_self(self) {
         drop(use_expr!(self));
     }
+
+    fn with_pat(use_pat!((a, b)): (u32, u32)) {
+        println!("Args: {} {}", a, b);
+    }
 }
 
 fn main() {}
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout b/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
new file mode 100644
index 00000000000..28812e20548
--- /dev/null
+++ b/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
@@ -0,0 +1,41 @@
+PRINT-BANG INPUT (DISPLAY): self
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "self",
+                span: $DIR/capture-macro-rules-invoke.rs:26:24: 26:28 (#CTXT),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:11:21: 11:26 (#CTXT),
+    },
+]
+PRINT-BANG INPUT (DISPLAY): (a, b)
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [
+                    Ident {
+                        ident: "a",
+                        span: $DIR/capture-macro-rules-invoke.rs:29:27: 29:28 (#CTXT),
+                    },
+                    Punct {
+                        ch: ',',
+                        spacing: Alone,
+                        span: $DIR/capture-macro-rules-invoke.rs:29:28: 29:29 (#CTXT),
+                    },
+                    Ident {
+                        ident: "b",
+                        span: $DIR/capture-macro-rules-invoke.rs:29:30: 29:31 (#CTXT),
+                    },
+                ],
+                span: $DIR/capture-macro-rules-invoke.rs:29:26: 29:32 (#CTXT),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:17:21: 17:25 (#CTXT),
+    },
+]
diff --git a/src/tools/clippy/clippy_lints/src/unnested_or_patterns.rs b/src/tools/clippy/clippy_lints/src/unnested_or_patterns.rs
index b7e2eba0a81..9fe771cef45 100644
--- a/src/tools/clippy/clippy_lints/src/unnested_or_patterns.rs
+++ b/src/tools/clippy/clippy_lints/src/unnested_or_patterns.rs
@@ -340,6 +340,7 @@ fn take_pat(from: &mut Pat) -> Pat {
         id: DUMMY_NODE_ID,
         kind: Wild,
         span: DUMMY_SP,
+        tokens: None
     };
     mem::replace(from, dummy)
 }