Attach tokens to `ast::Stmt`

We currently only attach tokens when parsing a `:stmt` matcher for a
`macro_rules!` macro. Proc-macro attributes on statements are still
unstable, and need additional work.

3 files changed, 22 insertions, 5 deletions

diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
index 3807daa7015..dedb9850b5a 100644
--- a/compiler/rustc_parse/src/lib.rs
+++ b/compiler/rustc_parse/src/lib.rs
@@ -269,6 +269,13 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
             prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
         }
         Nonterminal::NtBlock(ref block) => block.tokens.clone(),
+        Nonterminal::NtStmt(ref stmt) => {
+            // FIXME: We currently only collect tokens for `:stmt`
+            // matchers in `macro_rules!` macros. When we start collecting
+            // tokens for attributes on statements, we will need to prepend
+            // attributes here
+            stmt.tokens.clone()
+        }
         Nonterminal::NtPat(ref pat) => pat.tokens.clone(),
         Nonterminal::NtTy(ref ty) => ty.tokens.clone(),
         Nonterminal::NtIdent(ident, is_raw) => {
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
index f98f2a89dc6..15660fd574c 100644
--- a/compiler/rustc_parse/src/parser/nonterminal.rs
+++ b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -119,10 +119,20 @@ impl<'a> Parser<'a> {
                 }
                 token::NtBlock(block)
             }
-            NonterminalKind::Stmt => match self.parse_stmt()? {
-                Some(s) => token::NtStmt(s),
-                None => return Err(self.struct_span_err(self.token.span, "expected a statement")),
-            },
+            NonterminalKind::Stmt => {
+                let (stmt, tokens) = self.collect_tokens(|this| this.parse_stmt())?;
+                match stmt {
+                    Some(mut s) => {
+                        if s.tokens.is_none() {
+                            s.tokens = Some(tokens);
+                        }
+                        token::NtStmt(s)
+                    }
+                    None => {
+                        return Err(self.struct_span_err(self.token.span, "expected a statement"));
+                    }
+                }
+            }
             NonterminalKind::Pat => {
                 let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
                 // We have have eaten an NtPat, which could already have tokens
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index 6cc42487684..64b959e8325 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -415,7 +415,7 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn mk_stmt(&self, span: Span, kind: StmtKind) -> Stmt {
-        Stmt { id: DUMMY_NODE_ID, kind, span }
+        Stmt { id: DUMMY_NODE_ID, kind, span, tokens: None }
     }
 
     fn mk_stmt_err(&self, span: Span) -> Stmt {