Auto merge of #75800 - Aaron1011:feature/full-nt-tokens, r=petrochenkov

Attach tokens to all AST types used in `Nonterminal`

We perform token capturing when we have outer attributes (for nonterminals that support attributes - e.g. `Stmt`), or when we parse a `Nonterminal` for a `macro_rules!` argument. The full list of `Nonterminals` affected by this PR is:

* `NtBlock`
* `NtStmt`
* `NtTy`
* `NtMeta`
* `NtPath`
* `NtVis`
* `NtLiteral`

Of these nonterminals, only `NtStmt` and `NtLiteral` (which is actually just an `Expr`), support outer attributes - the rest only ever have token capturing perform when they match a `macro_rules!` argument.

This makes progress towards solving https://github.com/rust-lang/rust/issues/43081 - we now collect tokens for everything that might need them. However, we still need to handle `#[cfg]`, inner attributes, and misc pretty-printing issues (e.g. #75734)

I've separated the changes into (mostly) independent commits, which could be split into individual PRs for each `Nonterminal` variant. The purpose of having them all in one PR is to do a single Crater run for all of them.

Most of the changes in this PR are trivial (adding `tokens: None` everywhere we construct the various AST structs). The significant changes are:

* `ast::Visibility` is changed from `type Visibility = Spanned<VisibilityKind>` to a `struct Visibility { kind, span, tokens }`.
* `maybe_collect_tokens` is made generic, and used for both `ast::Expr` and `ast::Stmt`.
* Some of the statement-parsing functions are refactored so that we can capture the trailing semicolon.
* `Nonterminal` and `Expr` both grew by 8 bytes, as some of the structs which are stored inline (rather than behind a `P`) now have an `Option<TokenStream>` field. Hopefully the performance impact of doing this is negligible.

8 files changed, 121 insertions, 33 deletions

diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs
index 4e4429e461f..98f94098bfc 100644
--- a/compiler/rustc_parse/src/parser/attr.rs
+++ b/compiler/rustc_parse/src/parser/attr.rs
@@ -162,7 +162,7 @@ impl<'a> Parser<'a> {
         } else {
             let path = self.parse_path(PathStyle::Mod)?;
             let args = self.parse_attr_args()?;
-            ast::AttrItem { path, args }
+            ast::AttrItem { path, args, tokens: None }
         })
     }
 
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 12efe391fb9..e2a735188f9 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -28,7 +28,7 @@ pub(super) fn dummy_arg(ident: Ident) -> Param {
         span: ident.span,
         tokens: None,
     });
-    let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID };
+    let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID, tokens: None };
     Param {
         attrs: AttrVec::default(),
         id: ast::DUMMY_NODE_ID,
@@ -75,7 +75,12 @@ impl RecoverQPath for Ty {
         Some(P(self.clone()))
     }
     fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self {
-        Self { span: path.span, kind: TyKind::Path(qself, path), id: ast::DUMMY_NODE_ID }
+        Self {
+            span: path.span,
+            kind: TyKind::Path(qself, path),
+            id: ast::DUMMY_NODE_ID,
+            tokens: None,
+        }
     }
 }
 
@@ -896,7 +901,7 @@ impl<'a> Parser<'a> {
     ) -> PResult<'a, P<T>> {
         self.expect(&token::ModSep)?;
 
-        let mut path = ast::Path { segments: Vec::new(), span: DUMMY_SP };
+        let mut path = ast::Path { segments: Vec::new(), span: DUMMY_SP, tokens: None };
         self.parse_path_segments(&mut path.segments, T::PATH_STYLE)?;
         path.span = ty_span.to(self.prev_token.span);
 
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 1a428f8bb0a..26ca9980127 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -187,7 +187,7 @@ impl<'a> Parser<'a> {
 
     /// Error in-case a non-inherited visibility was parsed but no item followed.
     fn error_on_unmatched_vis(&self, vis: &Visibility) {
-        if let VisibilityKind::Inherited = vis.node {
+        if let VisibilityKind::Inherited = vis.kind {
             return;
         }
         let vs = pprust::vis_to_string(&vis);
@@ -296,7 +296,7 @@ impl<'a> Parser<'a> {
         } else if self.is_macro_rules_item() {
             // MACRO_RULES ITEM
             self.parse_item_macro_rules(vis)?
-        } else if vis.node.is_pub() && self.isnt_macro_invocation() {
+        } else if vis.kind.is_pub() && self.isnt_macro_invocation() {
             self.recover_missing_kw_before_item()?;
             return Ok(None);
         } else if macros_allowed && self.check_path() {
@@ -510,7 +510,12 @@ impl<'a> Parser<'a> {
         {
             let span = self.prev_token.span.between(self.token.span);
             self.struct_span_err(span, "missing trait in a trait impl").emit();
-            P(Ty { kind: TyKind::Path(None, err_path(span)), span, id: DUMMY_NODE_ID })
+            P(Ty {
+                kind: TyKind::Path(None, err_path(span)),
+                span,
+                id: DUMMY_NODE_ID,
+                tokens: None,
+            })
         } else {
             self.parse_ty()?
         };
@@ -782,7 +787,7 @@ impl<'a> Parser<'a> {
     fn parse_use_tree(&mut self) -> PResult<'a, UseTree> {
         let lo = self.token.span;
 
-        let mut prefix = ast::Path { segments: Vec::new(), span: lo.shrink_to_lo() };
+        let mut prefix = ast::Path { segments: Vec::new(), span: lo.shrink_to_lo(), tokens: None };
         let kind = if self.check(&token::OpenDelim(token::Brace))
             || self.check(&token::BinOp(token::Star))
             || self.is_import_coupler()
@@ -1046,7 +1051,7 @@ impl<'a> Parser<'a> {
 
         // The user intended that the type be inferred,
         // so treat this as if the user wrote e.g. `const A: _ = expr;`.
-        P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID })
+        P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID, tokens: None })
     }
 
     /// Parses an enum declaration.
@@ -1413,7 +1418,7 @@ impl<'a> Parser<'a> {
     /// Item macro invocations or `macro_rules!` definitions need inherited visibility.
     /// If that's not the case, emit an error.
     fn complain_if_pub_macro(&self, vis: &Visibility, macro_rules: bool) {
-        if let VisibilityKind::Inherited = vis.node {
+        if let VisibilityKind::Inherited = vis.kind {
             return;
         }
 
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index d22d08cd144..5eefae3af60 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -22,7 +22,7 @@ use rustc_ast::{Async, MacArgs, MacDelimiter, Mutability, StrLit, Visibility, Vi
 use rustc_ast_pretty::pprust;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError, PResult};
 use rustc_session::parse::ParseSess;
-use rustc_span::source_map::{respan, Span, DUMMY_SP};
+use rustc_span::source_map::{Span, DUMMY_SP};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use tracing::debug;
 
@@ -1022,14 +1022,22 @@ impl<'a> Parser<'a> {
         if self.is_crate_vis() {
             self.bump(); // `crate`
             self.sess.gated_spans.gate(sym::crate_visibility_modifier, self.prev_token.span);
-            return Ok(respan(self.prev_token.span, VisibilityKind::Crate(CrateSugar::JustCrate)));
+            return Ok(Visibility {
+                span: self.prev_token.span,
+                kind: VisibilityKind::Crate(CrateSugar::JustCrate),
+                tokens: None,
+            });
         }
 
         if !self.eat_keyword(kw::Pub) {
             // We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
             // keyword to grab a span from for inherited visibility; an empty span at the
             // beginning of the current token would seem to be the "Schelling span".
-            return Ok(respan(self.token.span.shrink_to_lo(), VisibilityKind::Inherited));
+            return Ok(Visibility {
+                span: self.token.span.shrink_to_lo(),
+                kind: VisibilityKind::Inherited,
+                tokens: None,
+            });
         }
         let lo = self.prev_token.span;
 
@@ -1046,7 +1054,11 @@ impl<'a> Parser<'a> {
                 self.bump(); // `crate`
                 self.expect(&token::CloseDelim(token::Paren))?; // `)`
                 let vis = VisibilityKind::Crate(CrateSugar::PubCrate);
-                return Ok(respan(lo.to(self.prev_token.span), vis));
+                return Ok(Visibility {
+                    span: lo.to(self.prev_token.span),
+                    kind: vis,
+                    tokens: None,
+                });
             } else if self.is_keyword_ahead(1, &[kw::In]) {
                 // Parse `pub(in path)`.
                 self.bump(); // `(`
@@ -1054,7 +1066,11 @@ impl<'a> Parser<'a> {
                 let path = self.parse_path(PathStyle::Mod)?; // `path`
                 self.expect(&token::CloseDelim(token::Paren))?; // `)`
                 let vis = VisibilityKind::Restricted { path: P(path), id: ast::DUMMY_NODE_ID };
-                return Ok(respan(lo.to(self.prev_token.span), vis));
+                return Ok(Visibility {
+                    span: lo.to(self.prev_token.span),
+                    kind: vis,
+                    tokens: None,
+                });
             } else if self.look_ahead(2, |t| t == &token::CloseDelim(token::Paren))
                 && self.is_keyword_ahead(1, &[kw::Super, kw::SelfLower])
             {
@@ -1063,7 +1079,11 @@ impl<'a> Parser<'a> {
                 let path = self.parse_path(PathStyle::Mod)?; // `super`/`self`
                 self.expect(&token::CloseDelim(token::Paren))?; // `)`
                 let vis = VisibilityKind::Restricted { path: P(path), id: ast::DUMMY_NODE_ID };
-                return Ok(respan(lo.to(self.prev_token.span), vis));
+                return Ok(Visibility {
+                    span: lo.to(self.prev_token.span),
+                    kind: vis,
+                    tokens: None,
+                });
             } else if let FollowedByType::No = fbt {
                 // Provide this diagnostic if a type cannot follow;
                 // in particular, if this is not a tuple struct.
@@ -1072,7 +1092,7 @@ impl<'a> Parser<'a> {
             }
         }
 
-        Ok(respan(lo, VisibilityKind::Public))
+        Ok(Visibility { span: lo, kind: VisibilityKind::Public, tokens: None })
     }
 
     /// Recovery for e.g. `pub(something) fn ...` or `struct X { pub(something) y: Z }`
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
index f40cd1131d2..15660fd574c 100644
--- a/compiler/rustc_parse/src/parser/nonterminal.rs
+++ b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -111,11 +111,28 @@ impl<'a> Parser<'a> {
                     return Err(self.struct_span_err(self.token.span, "expected an item keyword"));
                 }
             },
-            NonterminalKind::Block => token::NtBlock(self.parse_block()?),
-            NonterminalKind::Stmt => match self.parse_stmt()? {
-                Some(s) => token::NtStmt(s),
-                None => return Err(self.struct_span_err(self.token.span, "expected a statement")),
-            },
+            NonterminalKind::Block => {
+                let (mut block, tokens) = self.collect_tokens(|this| this.parse_block())?;
+                // We have have eaten an NtBlock, which could already have tokens
+                if block.tokens.is_none() {
+                    block.tokens = Some(tokens);
+                }
+                token::NtBlock(block)
+            }
+            NonterminalKind::Stmt => {
+                let (stmt, tokens) = self.collect_tokens(|this| this.parse_stmt())?;
+                match stmt {
+                    Some(mut s) => {
+                        if s.tokens.is_none() {
+                            s.tokens = Some(tokens);
+                        }
+                        token::NtStmt(s)
+                    }
+                    None => {
+                        return Err(self.struct_span_err(self.token.span, "expected a statement"));
+                    }
+                }
+            }
             NonterminalKind::Pat => {
                 let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
                 // We have have eaten an NtPat, which could already have tokens
@@ -133,8 +150,23 @@ impl<'a> Parser<'a> {
                 }
                 token::NtExpr(expr)
             }
-            NonterminalKind::Literal => token::NtLiteral(self.parse_literal_maybe_minus()?),
-            NonterminalKind::Ty => token::NtTy(self.parse_ty()?),
+            NonterminalKind::Literal => {
+                let (mut lit, tokens) =
+                    self.collect_tokens(|this| this.parse_literal_maybe_minus())?;
+                // We have have eaten a nonterminal, which  could already have tokens
+                if lit.tokens.is_none() {
+                    lit.tokens = Some(tokens);
+                }
+                token::NtLiteral(lit)
+            }
+            NonterminalKind::Ty => {
+                let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?;
+                // We have an eaten an NtTy, which could already have tokens
+                if ty.tokens.is_none() {
+                    ty.tokens = Some(tokens);
+                }
+                token::NtTy(ty)
+            }
             // this could be handled like a token, since it is one
             NonterminalKind::Ident => {
                 if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
@@ -146,10 +178,33 @@ impl<'a> Parser<'a> {
                     return Err(self.struct_span_err(self.token.span, msg));
                 }
             }
-            NonterminalKind::Path => token::NtPath(self.parse_path(PathStyle::Type)?),
-            NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item()?)),
+            NonterminalKind::Path => {
+                let (mut path, tokens) =
+                    self.collect_tokens(|this| this.parse_path(PathStyle::Type))?;
+                // We have have eaten an NtPath, which could already have tokens
+                if path.tokens.is_none() {
+                    path.tokens = Some(tokens);
+                }
+                token::NtPath(path)
+            }
+            NonterminalKind::Meta => {
+                let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?;
+                // We may have eaten a nonterminal, which could already have tokens
+                if attr.tokens.is_none() {
+                    attr.tokens = Some(tokens);
+                }
+                token::NtMeta(P(attr))
+            }
             NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
-            NonterminalKind::Vis => token::NtVis(self.parse_visibility(FollowedByType::Yes)?),
+            NonterminalKind::Vis => {
+                let (mut vis, tokens) =
+                    self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?;
+                // We may have etan an `NtVis`, which could already have tokens
+                if vis.tokens.is_none() {
+                    vis.tokens = Some(tokens);
+                }
+                token::NtVis(vis)
+            }
             NonterminalKind::Lifetime => {
                 if self.check_lifetime() {
                     token::NtLifetime(self.expect_lifetime().ident)
diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs
index 54b4df8613f..66ce015d02e 100644
--- a/compiler/rustc_parse/src/parser/path.rs
+++ b/compiler/rustc_parse/src/parser/path.rs
@@ -64,7 +64,7 @@ impl<'a> Parser<'a> {
             path_span = path_lo.to(self.prev_token.span);
         } else {
             path_span = self.token.span.to(self.token.span);
-            path = ast::Path { segments: Vec::new(), span: path_span };
+            path = ast::Path { segments: Vec::new(), span: path_span, tokens: None };
         }
 
         // See doc comment for `unmatched_angle_bracket_count`.
@@ -81,7 +81,10 @@ impl<'a> Parser<'a> {
         let qself = QSelf { ty, path_span, position: path.segments.len() };
         self.parse_path_segments(&mut path.segments, style)?;
 
-        Ok((qself, Path { segments: path.segments, span: lo.to(self.prev_token.span) }))
+        Ok((
+            qself,
+            Path { segments: path.segments, span: lo.to(self.prev_token.span), tokens: None },
+        ))
     }
 
     /// Recover from an invalid single colon, when the user likely meant a qualified path.
@@ -144,7 +147,7 @@ impl<'a> Parser<'a> {
         }
         self.parse_path_segments(&mut segments, style)?;
 
-        Ok(Path { segments, span: lo.to(self.prev_token.span) })
+        Ok(Path { segments, span: lo.to(self.prev_token.span), tokens: None })
     }
 
     pub(super) fn parse_path_segments(
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index 947ca6b5bd5..64b959e8325 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -411,11 +411,11 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn mk_block(&self, stmts: Vec<Stmt>, rules: BlockCheckMode, span: Span) -> P<Block> {
-        P(Block { stmts, id: DUMMY_NODE_ID, rules, span })
+        P(Block { stmts, id: DUMMY_NODE_ID, rules, span, tokens: None })
     }
 
     pub(super) fn mk_stmt(&self, span: Span, kind: StmtKind) -> Stmt {
-        Stmt { id: DUMMY_NODE_ID, kind, span }
+        Stmt { id: DUMMY_NODE_ID, kind, span, tokens: None }
     }
 
     fn mk_stmt_err(&self, span: Span) -> Stmt {
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index 4356850818e..259764a317d 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -626,6 +626,6 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn mk_ty(&self, span: Span, kind: TyKind) -> P<Ty> {
-        P(Ty { kind, span, id: ast::DUMMY_NODE_ID })
+        P(Ty { kind, span, id: ast::DUMMY_NODE_ID, tokens: None })
     }
 }