Rollup merge of #66935 - petrochenkov:attrtok2, r=Centril

syntax: Unify macro and attribute arguments in AST

The unified form (`ast::MacArgs`) represents parsed arguments instead of an unstructured token stream that was previously used for attributes.
It also tracks some spans and delimiter kinds better for fn-like macros and macro definitions.

I've been talking about implementing this with @nnethercote in https://github.com/rust-lang/rust/pull/65750#issuecomment-546517322.
The parsed representation is closer to `MetaItem` and requires less token juggling during conversions, so it potentially may be faster.

r? @Centril

6 files changed, 20 insertions, 44 deletions

diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs
index a6ced1439c5..9bfedb3b617 100644
--- a/src/libsyntax_expand/expand.rs
+++ b/src/libsyntax_expand/expand.rs
@@ -11,7 +11,7 @@ use rustc_parse::DirectoryOwnership;
 use rustc_parse::parser::Parser;
 use rustc_parse::validate_attr;
 use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
-use syntax::ast::{MacStmtStyle, StmtKind, ItemKind};
+use syntax::ast::{MacArgs, MacStmtStyle, StmtKind, ItemKind};
 use syntax::attr::{self, HasAttrs, is_builtin_attr};
 use syntax::source_map::respan;
 use syntax::feature_gate::{self, feature_err};
@@ -597,13 +597,13 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             InvocationKind::Bang { mac, .. } => match ext {
                 SyntaxExtensionKind::Bang(expander) => {
                     self.gate_proc_macro_expansion_kind(span, fragment_kind);
-                    let tok_result = expander.expand(self.cx, span, mac.stream());
+                    let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens());
                     self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span)
                 }
                 SyntaxExtensionKind::LegacyBang(expander) => {
                     let prev = self.cx.current_expansion.prior_type_ascription;
                     self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription;
-                    let tok_result = expander.expand(self.cx, span, mac.stream());
+                    let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens());
                     let result = if let Some(result) = fragment_kind.make_from(tok_result) {
                         result
                     } else {
@@ -642,8 +642,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                             => panic!("unexpected annotatable"),
                     })), DUMMY_SP).into();
                     let item = attr.unwrap_normal_item();
-                    let input = self.extract_proc_macro_attr_input(item.tokens, span);
-                    let tok_result = expander.expand(self.cx, span, input, item_tok);
+                    if let MacArgs::Eq(..) = item.args {
+                        self.cx.span_err(span, "key-value macro attributes are not supported");
+                    }
+                    let tok_result =
+                        expander.expand(self.cx, span, item.args.inner_tokens(), item_tok);
                     self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span)
                 }
                 SyntaxExtensionKind::LegacyAttr(expander) => {
@@ -687,23 +690,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         }
     }
 
-    fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
-        let mut trees = tokens.trees();
-        match trees.next() {
-            Some(TokenTree::Delimited(_, _, tts)) => {
-                if trees.next().is_none() {
-                    return tts.into()
-                }
-            }
-            Some(TokenTree::Token(..)) => {}
-            None => return TokenStream::default(),
-        }
-        self.cx.span_err(span, "custom attribute invocations must be \
-            of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \
-            followed by a delimiter token");
-        TokenStream::default()
-    }
-
     fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) {
         let kind = match item {
             Annotatable::Item(item) => match &item.kind {
@@ -1560,7 +1546,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
             let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
             *at = attr::Attribute {
                 kind: ast::AttrKind::Normal(
-                    AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) },
+                    AttrItem { path: meta.path, args: meta.kind.mac_args(meta.span) },
                 ),
                 span: at.span,
                 id: at.id,
diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs
index b191527df19..e3c3655bcf8 100644
--- a/src/libsyntax_expand/mbe/macro_rules.rs
+++ b/src/libsyntax_expand/mbe/macro_rules.rs
@@ -318,8 +318,8 @@ pub fn compile_declarative_macro(
     let tt_spec = ast::Ident::new(sym::tt, def.span);
 
     // Parse the macro_rules! invocation
-    let body = match def.kind {
-        ast::ItemKind::MacroDef(ref body) => body,
+    let (is_legacy, body) = match &def.kind {
+        ast::ItemKind::MacroDef(macro_def) => (macro_def.legacy, macro_def.body.inner_tokens()),
         _ => unreachable!(),
     };
 
@@ -338,7 +338,7 @@ pub fn compile_declarative_macro(
                     mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec),
                 ],
                 separator: Some(Token::new(
-                    if body.legacy { token::Semi } else { token::Comma },
+                    if is_legacy { token::Semi } else { token::Comma },
                     def.span,
                 )),
                 kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span),
@@ -350,7 +350,7 @@ pub fn compile_declarative_macro(
             DelimSpan::dummy(),
             Lrc::new(mbe::SequenceRepetition {
                 tts: vec![mbe::TokenTree::token(
-                    if body.legacy { token::Semi } else { token::Comma },
+                    if is_legacy { token::Semi } else { token::Comma },
                     def.span,
                 )],
                 separator: None,
@@ -360,7 +360,7 @@ pub fn compile_declarative_macro(
         ),
     ];
 
-    let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) {
+    let argument_map = match parse(sess, body, &argument_gram, None, true) {
         Success(m) => m,
         Failure(token, msg) => {
             let s = parse_failure_msg(&token);
@@ -435,7 +435,7 @@ pub fn compile_declarative_macro(
     // that is not lint-checked and trigger the "failed to process buffered lint here" bug.
     valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses);
 
-    let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy);
+    let (transparency, transparency_error) = attr::find_transparency(&def.attrs, is_legacy);
     match transparency_error {
         Some(TransparencyError::UnknownTransparency(value, span)) =>
             diag.span_err(span, &format!("unknown macro transparency: `{}`", value)),
diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs
index 4092d4b97de..a1157667df1 100644
--- a/src/libsyntax_expand/mbe/transcribe.rs
+++ b/src/libsyntax_expand/mbe/transcribe.rs
@@ -30,13 +30,6 @@ impl MutVisitor for Marker {
     }
 }
 
-impl Marker {
-    fn visit_delim_span(&mut self, dspan: &mut DelimSpan) {
-        self.visit_span(&mut dspan.open);
-        self.visit_span(&mut dspan.close);
-    }
-}
-
 /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
 enum Frame {
     Delimited { forest: Lrc<mbe::Delimited>, idx: usize, span: DelimSpan },
@@ -271,7 +264,7 @@ pub(super) fn transcribe(
             // jump back out of the Delimited, pop the result_stack and add the new results back to
             // the previous results (from outside the Delimited).
             mbe::TokenTree::Delimited(mut span, delimited) => {
-                marker.visit_delim_span(&mut span);
+                mut_visit::visit_delim_span(&mut span, &mut marker);
                 stack.push(Frame::Delimited { forest: delimited, idx: 0, span });
                 result_stack.push(mem::take(&mut result));
             }
diff --git a/src/libsyntax_expand/parse/tests.rs b/src/libsyntax_expand/parse/tests.rs
index 08950ddefba..30e83c151e2 100644
--- a/src/libsyntax_expand/parse/tests.rs
+++ b/src/libsyntax_expand/parse/tests.rs
@@ -272,7 +272,7 @@ fn ttdelim_span() {
             "foo!( fn main() { body } )".to_string(), &sess).unwrap();
 
         let tts: Vec<_> = match expr.kind {
-            ast::ExprKind::Mac(ref mac) => mac.stream().trees().collect(),
+            ast::ExprKind::Mac(ref mac) => mac.args.inner_tokens().trees().collect(),
             _ => panic!("not a macro"),
         };
 
diff --git a/src/libsyntax_expand/placeholders.rs b/src/libsyntax_expand/placeholders.rs
index 6cbe8c13245..74ade1de20e 100644
--- a/src/libsyntax_expand/placeholders.rs
+++ b/src/libsyntax_expand/placeholders.rs
@@ -3,7 +3,6 @@ use crate::expand::{AstFragment, AstFragmentKind};
 
 use syntax::ast;
 use syntax::source_map::{DUMMY_SP, dummy_spanned};
-use syntax::tokenstream::TokenStream;
 use syntax::mut_visit::*;
 use syntax::ptr::P;
 use syntax::ThinVec;
@@ -17,9 +16,7 @@ pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId, vis: Option<ast::Visi
     fn mac_placeholder() -> ast::Mac {
         ast::Mac {
             path: ast::Path { span: DUMMY_SP, segments: Vec::new() },
-            tts: TokenStream::default().into(),
-            delim: ast::MacDelimiter::Brace,
-            span: DUMMY_SP,
+            args: P(ast::MacArgs::Empty),
             prior_type_ascription: None,
         }
     }
diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs
index 099cf0a4be9..8e56e2bb00b 100644
--- a/src/libsyntax_expand/proc_macro.rs
+++ b/src/libsyntax_expand/proc_macro.rs
@@ -1,7 +1,7 @@
 use crate::base::{self, *};
 use crate::proc_macro_server;
 
-use syntax::ast::{self, ItemKind};
+use syntax::ast::{self, ItemKind, MacArgs};
 use syntax::errors::{Applicability, FatalError};
 use syntax::symbol::sym;
 use syntax::token;
@@ -183,7 +183,7 @@ crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>)
         }
 
         let parse_derive_paths = |attr: &ast::Attribute| {
-            if attr.get_normal_item().tokens.is_empty() {
+            if let MacArgs::Empty = attr.get_normal_item().args {
                 return Ok(Vec::new());
             }
             rustc_parse::parse_in_attr(cx.parse_sess, attr, |p| p.parse_derive_paths())