Rollup merge of #66935 - petrochenkov:attrtok2, r=Centril

syntax: Unify macro and attribute arguments in AST

The unified form (`ast::MacArgs`) represents parsed arguments instead of an unstructured token stream that was previously used for attributes.
It also tracks some spans and delimiter kinds better for fn-like macros and macro definitions.

I've been talking about implementing this with @nnethercote in https://github.com/rust-lang/rust/pull/65750#issuecomment-546517322.
The parsed representation is closer to `MetaItem` and requires less token juggling during conversions, so it potentially may be faster.

r? @Centril

11 files changed, 100 insertions, 119 deletions

diff --git a/src/librustc_parse/config.rs b/src/librustc_parse/config.rs
index 26e51e83d62..1bf6e9ecbc0 100644
--- a/src/librustc_parse/config.rs
+++ b/src/librustc_parse/config.rs
@@ -101,7 +101,7 @@ impl<'a> StripUnconfigured<'a> {
         if !attr.has_name(sym::cfg_attr) {
             return vec![attr];
         }
-        if attr.get_normal_item().tokens.is_empty() {
+        if let ast::MacArgs::Empty = attr.get_normal_item().args {
             self.sess.span_diagnostic
                 .struct_span_err(
                     attr.span,
diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs
index 1215c7a199a..a22b383e5f3 100644
--- a/src/librustc_parse/lib.rs
+++ b/src/librustc_parse/lib.rs
@@ -277,7 +277,9 @@ pub fn parse_in_attr<'a, T>(
 ) -> PResult<'a, T> {
     let mut parser = Parser::new(
         sess,
-        attr.get_normal_item().tokens.clone(),
+        // FIXME(#66940, Centril | petrochenkov): refactor this function so it doesn't
+        // require reconstructing and immediately re-parsing delimiters.
+        attr.get_normal_item().args.outer_tokens(),
         None,
         false,
         false,
@@ -409,7 +411,7 @@ fn prepend_attrs(
             brackets.push(stream);
         }
 
-        brackets.push(item.tokens.clone());
+        brackets.push(item.args.outer_tokens());
 
         // The span we list here for `#` and for `[ ... ]` are both wrong in
         // that it encompasses more than each token, but it hopefully is "good
diff --git a/src/librustc_parse/parser/attr.rs b/src/librustc_parse/parser/attr.rs
index 524b551e54c..c7261404f54 100644
--- a/src/librustc_parse/parser/attr.rs
+++ b/src/librustc_parse/parser/attr.rs
@@ -2,8 +2,7 @@ use super::{SeqSep, Parser, TokenType, PathStyle};
 use syntax::attr;
 use syntax::ast;
 use syntax::util::comments;
-use syntax::token::{self, Nonterminal, DelimToken};
-use syntax::tokenstream::{TokenStream, TokenTree};
+use syntax::token::{self, Nonterminal};
 use syntax_pos::{Span, Symbol};
 use errors::PResult;
 
@@ -181,31 +180,8 @@ impl<'a> Parser<'a> {
             item
         } else {
             let path = self.parse_path(PathStyle::Mod)?;
-            let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) ||
-               self.check(&token::OpenDelim(DelimToken::Bracket)) ||
-               self.check(&token::OpenDelim(DelimToken::Brace)) {
-                   self.parse_token_tree().into()
-            } else if self.eat(&token::Eq) {
-                let eq = TokenTree::token(token::Eq, self.prev_span);
-                let mut is_interpolated_expr = false;
-                if let token::Interpolated(nt) = &self.token.kind {
-                    if let token::NtExpr(..) = **nt {
-                        is_interpolated_expr = true;
-                    }
-                }
-                let token_tree = if is_interpolated_expr {
-                    // We need to accept arbitrary interpolated expressions to continue
-                    // supporting things like `doc = $expr` that work on stable.
-                    // Non-literal interpolated expressions are rejected after expansion.
-                    self.parse_token_tree()
-                } else {
-                    self.parse_unsuffixed_lit()?.token_tree()
-                };
-                TokenStream::new(vec![eq.into(), token_tree.into()])
-            } else {
-                TokenStream::default()
-            };
-            ast::AttrItem { path, tokens }
+            let args = self.parse_attr_args()?;
+            ast::AttrItem { path, args }
         })
     }
 
@@ -244,7 +220,7 @@ impl<'a> Parser<'a> {
         Ok(attrs)
     }
 
-    fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
+    pub(super) fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
         let lit = self.parse_lit()?;
         debug!("checking if {:?} is unusuffixed", lit);
 
diff --git a/src/librustc_parse/parser/expr.rs b/src/librustc_parse/parser/expr.rs
index 43c740f7f93..1112274dc46 100644
--- a/src/librustc_parse/parser/expr.rs
+++ b/src/librustc_parse/parser/expr.rs
@@ -922,13 +922,11 @@ impl<'a> Parser<'a> {
                     // `!`, as an operator, is prefix, so we know this isn't that.
                     if self.eat(&token::Not) {
                         // MACRO INVOCATION expression
-                        let (delim, tts) = self.expect_delimited_token_tree()?;
+                        let args = self.parse_mac_args()?;
                         hi = self.prev_span;
                         ex = ExprKind::Mac(Mac {
                             path,
-                            tts,
-                            delim,
-                            span: lo.to(hi),
+                            args,
                             prior_type_ascription: self.last_type_ascription,
                         });
                     } else if self.check(&token::OpenDelim(token::Brace)) {
diff --git a/src/librustc_parse/parser/item.rs b/src/librustc_parse/parser/item.rs
index a0669a2a174..46addba57c6 100644
--- a/src/librustc_parse/parser/item.rs
+++ b/src/librustc_parse/parser/item.rs
@@ -8,12 +8,12 @@ use syntax::ast::{ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, Us
 use syntax::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness, Extern, StrLit};
 use syntax::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind};
 use syntax::ast::{Ty, TyKind, Generics, TraitRef, EnumDef, Variant, VariantData, StructField};
-use syntax::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
+use syntax::ast::{Mac, MacArgs, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::ThinVec;
 use syntax::token;
-use syntax::tokenstream::{TokenTree, TokenStream};
+use syntax::tokenstream::{DelimSpan, TokenTree, TokenStream};
 use syntax::source_map::{self, respan, Span};
 use syntax::struct_span_err;
 use syntax_pos::BytePos;
@@ -432,22 +432,18 @@ impl<'a> Parser<'a> {
             let prev_span = self.prev_span;
             self.complain_if_pub_macro(&visibility.node, prev_span);
 
-            let mac_lo = self.token.span;
-
             // Item macro
             let path = self.parse_path(PathStyle::Mod)?;
             self.expect(&token::Not)?;
-            let (delim, tts) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
+            let args = self.parse_mac_args()?;
+            if args.need_semicolon() && !self.eat(&token::Semi) {
                 self.report_invalid_macro_expansion_item();
             }
 
             let hi = self.prev_span;
             let mac = Mac {
                 path,
-                tts,
-                delim,
-                span: mac_lo.to(hi),
+                args,
                 prior_type_ascription: self.last_type_ascription,
             };
             let item =
@@ -500,7 +496,6 @@ impl<'a> Parser<'a> {
         if self.token.is_path_start() &&
                 !(self.is_async_fn() && self.token.span.rust_2015()) {
             let prev_span = self.prev_span;
-            let lo = self.token.span;
             let path = self.parse_path(PathStyle::Mod)?;
 
             if path.segments.len() == 1 {
@@ -518,16 +513,14 @@ impl<'a> Parser<'a> {
             *at_end = true;
 
             // eat a matched-delimiter token tree:
-            let (delim, tts) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace {
+            let args = self.parse_mac_args()?;
+            if args.need_semicolon() {
                 self.expect_semi()?;
             }
 
             Ok(Some(Mac {
                 path,
-                tts,
-                delim,
-                span: lo.to(self.prev_span),
+                args,
                 prior_type_ascription: self.last_type_ascription,
             }))
         } else {
@@ -1624,33 +1617,31 @@ impl<'a> Parser<'a> {
         vis: &Visibility,
         lo: Span
     ) -> PResult<'a, Option<P<Item>>> {
-        let token_lo = self.token.span;
         let (ident, def) = if self.eat_keyword(kw::Macro) {
             let ident = self.parse_ident()?;
-            let tokens = if self.check(&token::OpenDelim(token::Brace)) {
-                match self.parse_token_tree() {
-                    TokenTree::Delimited(_, _, tts) => tts,
-                    _ => unreachable!(),
-                }
+            let body = if self.check(&token::OpenDelim(token::Brace)) {
+                self.parse_mac_args()?
             } else if self.check(&token::OpenDelim(token::Paren)) {
-                let args = self.parse_token_tree();
+                let params = self.parse_token_tree();
+                let pspan = params.span();
                 let body = if self.check(&token::OpenDelim(token::Brace)) {
                     self.parse_token_tree()
                 } else {
-                    self.unexpected()?;
-                    unreachable!()
+                    return self.unexpected();
                 };
-                TokenStream::new(vec![
-                    args.into(),
-                    TokenTree::token(token::FatArrow, token_lo.to(self.prev_span)).into(),
+                let bspan = body.span();
+                let tokens = TokenStream::new(vec![
+                    params.into(),
+                    TokenTree::token(token::FatArrow, pspan.between(bspan)).into(),
                     body.into(),
-                ])
+                ]);
+                let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
+                P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens))
             } else {
-                self.unexpected()?;
-                unreachable!()
+                return self.unexpected();
             };
 
-            (ident, ast::MacroDef { tokens: tokens.into(), legacy: false })
+            (ident, ast::MacroDef { body, legacy: false })
         } else if self.check_keyword(sym::macro_rules) &&
                   self.look_ahead(1, |t| *t == token::Not) &&
                   self.look_ahead(2, |t| t.is_ident()) {
@@ -1660,12 +1651,12 @@ impl<'a> Parser<'a> {
             self.bump();
 
             let ident = self.parse_ident()?;
-            let (delim, tokens) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
+            let body = self.parse_mac_args()?;
+            if body.need_semicolon() && !self.eat(&token::Semi) {
                 self.report_invalid_macro_expansion_item();
             }
 
-            (ident, ast::MacroDef { tokens, legacy: true })
+            (ident, ast::MacroDef { body, legacy: true })
         } else {
             return Ok(None);
         };
diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs
index ea7673767d0..28689720044 100644
--- a/src/librustc_parse/parser/mod.rs
+++ b/src/librustc_parse/parser/mod.rs
@@ -16,7 +16,7 @@ use crate::lexer::UnmatchedBrace;
 
 use syntax::ast::{
     self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Extern, Ident, StrLit,
-    IsAsync, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
+    IsAsync, MacArgs, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
 };
 
 use syntax::print::pprust;
@@ -1010,27 +1010,49 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> {
-        let delim = match self.token.kind {
-            token::OpenDelim(delim) => delim,
-            _ => {
-                let msg = "expected open delimiter";
-                let mut err = self.fatal(msg);
-                err.span_label(self.token.span, msg);
-                return Err(err)
+    fn parse_mac_args(&mut self) -> PResult<'a, P<MacArgs>> {
+        self.parse_mac_args_common(true).map(P)
+    }
+
+    fn parse_attr_args(&mut self) -> PResult<'a, MacArgs> {
+        self.parse_mac_args_common(false)
+    }
+
+    fn parse_mac_args_common(&mut self, delimited_only: bool) -> PResult<'a, MacArgs> {
+        Ok(if self.check(&token::OpenDelim(DelimToken::Paren)) ||
+                       self.check(&token::OpenDelim(DelimToken::Bracket)) ||
+                       self.check(&token::OpenDelim(DelimToken::Brace)) {
+            match self.parse_token_tree() {
+                TokenTree::Delimited(dspan, delim, tokens) =>
+                    // We've confirmed above that there is a delimiter so unwrapping is OK.
+                    MacArgs::Delimited(dspan, MacDelimiter::from_token(delim).unwrap(), tokens),
+                _ => unreachable!(),
             }
-        };
-        let tts = match self.parse_token_tree() {
-            TokenTree::Delimited(_, _, tts) => tts,
-            _ => unreachable!(),
-        };
-        let delim = match delim {
-            token::Paren => MacDelimiter::Parenthesis,
-            token::Bracket => MacDelimiter::Bracket,
-            token::Brace => MacDelimiter::Brace,
-            token::NoDelim => self.bug("unexpected no delimiter"),
-        };
-        Ok((delim, tts.into()))
+        } else if !delimited_only {
+            if self.eat(&token::Eq) {
+                let eq_span = self.prev_span;
+                let mut is_interpolated_expr = false;
+                if let token::Interpolated(nt) = &self.token.kind {
+                    if let token::NtExpr(..) = **nt {
+                        is_interpolated_expr = true;
+                    }
+                }
+                let token_tree = if is_interpolated_expr {
+                    // We need to accept arbitrary interpolated expressions to continue
+                    // supporting things like `doc = $expr` that work on stable.
+                    // Non-literal interpolated expressions are rejected after expansion.
+                    self.parse_token_tree()
+                } else {
+                    self.parse_unsuffixed_lit()?.token_tree()
+                };
+
+                MacArgs::Eq(eq_span, token_tree.into())
+            } else {
+                MacArgs::Empty
+            }
+        } else {
+            return self.unexpected();
+        })
     }
 
     fn parse_or_use_outer_attributes(
diff --git a/src/librustc_parse/parser/pat.rs b/src/librustc_parse/parser/pat.rs
index b068a4f16a5..1127c4b2d5f 100644
--- a/src/librustc_parse/parser/pat.rs
+++ b/src/librustc_parse/parser/pat.rs
@@ -338,7 +338,7 @@ impl<'a> Parser<'a> {
                     (None, self.parse_path(PathStyle::Expr)?)
                 };
                 match self.token.kind {
-                    token::Not if qself.is_none() => self.parse_pat_mac_invoc(lo, path)?,
+                    token::Not if qself.is_none() => self.parse_pat_mac_invoc(path)?,
                     token::DotDotDot | token::DotDotEq | token::DotDot => {
                         self.parse_pat_range_starting_with_path(lo, qself, path)?
                     }
@@ -593,14 +593,12 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse macro invocation
-    fn parse_pat_mac_invoc(&mut self, lo: Span, path: Path) -> PResult<'a, PatKind> {
+    fn parse_pat_mac_invoc(&mut self, path: Path) -> PResult<'a, PatKind> {
         self.bump();
-        let (delim, tts) = self.expect_delimited_token_tree()?;
+        let args = self.parse_mac_args()?;
         let mac = Mac {
             path,
-            tts,
-            delim,
-            span: lo.to(self.prev_span),
+            args,
             prior_type_ascription: self.last_type_ascription,
         };
         Ok(PatKind::Mac(mac))
diff --git a/src/librustc_parse/parser/path.rs b/src/librustc_parse/parser/path.rs
index 68307440712..75bb67d47bc 100644
--- a/src/librustc_parse/parser/path.rs
+++ b/src/librustc_parse/parser/path.rs
@@ -2,6 +2,7 @@ use super::{Parser, TokenType};
 use crate::maybe_whole;
 use syntax::ast::{self, QSelf, Path, PathSegment, Ident, ParenthesizedArgs, AngleBracketedArgs};
 use syntax::ast::{AnonConst, GenericArg, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode};
+use syntax::ast::MacArgs;
 use syntax::ThinVec;
 use syntax::token::{self, Token};
 use syntax::source_map::{Span, BytePos};
@@ -114,9 +115,9 @@ impl<'a> Parser<'a> {
     fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> {
         let meta_ident = match self.token.kind {
             token::Interpolated(ref nt) => match **nt {
-                token::NtMeta(ref item) => match item.tokens.is_empty() {
-                    true => Some(item.path.clone()),
-                    false => None,
+                token::NtMeta(ref item) => match item.args {
+                    MacArgs::Empty => Some(item.path.clone()),
+                    _ => None,
                 },
                 _ => None,
             },
diff --git a/src/librustc_parse/parser/stmt.rs b/src/librustc_parse/parser/stmt.rs
index a5f20691d07..b952e8814a3 100644
--- a/src/librustc_parse/parser/stmt.rs
+++ b/src/librustc_parse/parser/stmt.rs
@@ -10,7 +10,7 @@ use syntax::ThinVec;
 use syntax::ptr::P;
 use syntax::ast;
 use syntax::ast::{DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind};
-use syntax::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter};
+use syntax::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac};
 use syntax::util::classify;
 use syntax::token;
 use syntax::source_map::{respan, Span};
@@ -93,10 +93,11 @@ impl<'a> Parser<'a> {
                 }));
             }
 
-            let (delim, tts) = self.expect_delimited_token_tree()?;
+            let args = self.parse_mac_args()?;
+            let delim = args.delim();
             let hi = self.prev_span;
 
-            let style = if delim == MacDelimiter::Brace {
+            let style = if delim == token::Brace {
                 MacStmtStyle::Braces
             } else {
                 MacStmtStyle::NoBraces
@@ -104,12 +105,10 @@ impl<'a> Parser<'a> {
 
             let mac = Mac {
                 path,
-                tts,
-                delim,
-                span: lo.to(hi),
+                args,
                 prior_type_ascription: self.last_type_ascription,
             };
-            let kind = if delim == MacDelimiter::Brace ||
+            let kind = if delim == token::Brace ||
                           self.token == token::Semi || self.token == token::Eof {
                 StmtKind::Mac(P((mac, style, attrs.into())))
             }
@@ -130,7 +129,7 @@ impl<'a> Parser<'a> {
                 self.warn_missing_semicolon();
                 StmtKind::Mac(P((mac, style, attrs.into())))
             } else {
-                let e = self.mk_expr(mac.span, ExprKind::Mac(mac), ThinVec::new());
+                let e = self.mk_expr(lo.to(hi), ExprKind::Mac(mac), ThinVec::new());
                 let e = self.maybe_recover_from_bad_qpath(e, true)?;
                 let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
                 let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
diff --git a/src/librustc_parse/parser/ty.rs b/src/librustc_parse/parser/ty.rs
index 8e6bc29be52..32142796905 100644
--- a/src/librustc_parse/parser/ty.rs
+++ b/src/librustc_parse/parser/ty.rs
@@ -177,12 +177,10 @@ impl<'a> Parser<'a> {
             let path = self.parse_path(PathStyle::Type)?;
             if self.eat(&token::Not) {
                 // Macro invocation in type position
-                let (delim, tts) = self.expect_delimited_token_tree()?;
+                let args = self.parse_mac_args()?;
                 let mac = Mac {
                     path,
-                    tts,
-                    delim,
-                    span: lo.to(self.prev_span),
+                    args,
                     prior_type_ascription: self.last_type_ascription,
                 };
                 TyKind::Mac(mac)
diff --git a/src/librustc_parse/validate_attr.rs b/src/librustc_parse/validate_attr.rs
index a3c9e266593..0fb348efece 100644
--- a/src/librustc_parse/validate_attr.rs
+++ b/src/librustc_parse/validate_attr.rs
@@ -2,11 +2,9 @@
 
 use errors::{PResult, Applicability};
 use rustc_feature::{AttributeTemplate, BUILTIN_ATTRIBUTE_MAP};
-use syntax::ast::{self, Attribute, AttrKind, Ident, MetaItem, MetaItemKind};
+use syntax::ast::{self, Attribute, AttrKind, Ident, MacArgs, MetaItem, MetaItemKind};
 use syntax::attr::mk_name_value_item_str;
 use syntax::early_buffered_lints::BufferedEarlyLintId;
-use syntax::token;
-use syntax::tokenstream::TokenTree;
 use syntax::sess::ParseSess;
 use syntax_pos::{Symbol, sym};
 
@@ -19,11 +17,9 @@ pub fn check_meta(sess: &ParseSess, attr: &Attribute) {
         // `rustc_dummy` doesn't have any restrictions specific to built-in attributes.
         Some((name, _, template, _)) if name != sym::rustc_dummy =>
             check_builtin_attribute(sess, attr, name, template),
-        _ => if let Some(TokenTree::Token(token)) = attr.get_normal_item().tokens.trees().next() {
-            if token == token::Eq {
-                // All key-value attributes are restricted to meta-item syntax.
-                parse_meta(sess, attr).map_err(|mut err| err.emit()).ok();
-            }
+        _ => if let MacArgs::Eq(..) = attr.get_normal_item().args {
+            // All key-value attributes are restricted to meta-item syntax.
+            parse_meta(sess, attr).map_err(|mut err| err.emit()).ok();
         }
     }
 }