Rollup merge of #66935 - petrochenkov:attrtok2, r=Centril

syntax: Unify macro and attribute arguments in AST

The unified form (`ast::MacArgs`) represents parsed arguments instead of an unstructured token stream that was previously used for attributes.
It also tracks some spans and delimiter kinds better for fn-like macros and macro definitions.

I've been talking about implementing this with @nnethercote in https://github.com/rust-lang/rust/pull/65750#issuecomment-546517322.
The parsed representation is closer to `MetaItem` and requires less token juggling during conversions, so it potentially may be faster.

r? @Centril

3 files changed, 17 insertions, 26 deletions

diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs
index c4f3c03813f..c788d062994 100644
--- a/src/libsyntax_ext/assert.rs
+++ b/src/libsyntax_ext/assert.rs
@@ -6,7 +6,7 @@ use syntax::token::{self, TokenKind};
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::symbol::{sym, Symbol};
-use syntax::tokenstream::{TokenStream, TokenTree};
+use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use syntax_expand::base::*;
 use syntax_pos::{Span, DUMMY_SP};
 
@@ -26,19 +26,19 @@ pub fn expand_assert<'cx>(
     // `core::panic` and `std::panic` are different macros, so we use call-site
     // context to pick up whichever is currently in scope.
     let sp = cx.with_call_site_ctxt(sp);
+    let tokens = custom_message.unwrap_or_else(|| {
+        TokenStream::from(TokenTree::token(
+            TokenKind::lit(token::Str, Symbol::intern(&format!(
+                "assertion failed: {}",
+                pprust::expr_to_string(&cond_expr).escape_debug()
+            )), None),
+            DUMMY_SP,
+        ))
+    });
+    let args = P(MacArgs::Delimited(DelimSpan::from_single(sp), MacDelimiter::Parenthesis, tokens));
     let panic_call = Mac {
         path: Path::from_ident(Ident::new(sym::panic, sp)),
-        tts: custom_message.unwrap_or_else(|| {
-            TokenStream::from(TokenTree::token(
-                TokenKind::lit(token::Str, Symbol::intern(&format!(
-                    "assertion failed: {}",
-                    pprust::expr_to_string(&cond_expr).escape_debug()
-                )), None),
-                DUMMY_SP,
-            ))
-        }).into(),
-        delim: MacDelimiter::Parenthesis,
-        span: sp,
+        args,
         prior_type_ascription: None,
     };
     let if_expr = cx.expr_if(
diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs
index 5c3009c432c..98cf8a34742 100644
--- a/src/libsyntax_ext/cmdline_attrs.rs
+++ b/src/libsyntax_ext/cmdline_attrs.rs
@@ -16,7 +16,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.token.span;
-        let AttrItem { path, tokens } = panictry!(parser.parse_attr_item());
+        let AttrItem { path, args } = panictry!(parser.parse_attr_item());
         let end_span = parser.token.span;
         if parser.token != token::Eof {
             parse_sess.span_diagnostic
@@ -24,7 +24,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
             continue;
         }
 
-        krate.attrs.push(mk_attr(AttrStyle::Inner, path, tokens, start_span.to(end_span)));
+        krate.attrs.push(mk_attr(AttrStyle::Inner, path, args, start_span.to(end_span)));
     }
 
     krate
diff --git a/src/libsyntax_ext/deriving/generic/mod.rs b/src/libsyntax_ext/deriving/generic/mod.rs
index b6bf2f88161..5bd84b43a78 100644
--- a/src/libsyntax_ext/deriving/generic/mod.rs
+++ b/src/libsyntax_ext/deriving/generic/mod.rs
@@ -340,14 +340,12 @@ pub fn combine_substructure(f: CombineSubstructureFunc<'_>)
 fn find_type_parameters(
     ty: &ast::Ty,
     ty_param_names: &[ast::Name],
-    span: Span,
     cx: &ExtCtxt<'_>,
 ) -> Vec<P<ast::Ty>> {
     use syntax::visit;
 
     struct Visitor<'a, 'b> {
         cx: &'a ExtCtxt<'b>,
-        span: Span,
         ty_param_names: &'a [ast::Name],
         types: Vec<P<ast::Ty>>,
     }
@@ -366,18 +364,11 @@ fn find_type_parameters(
         }
 
         fn visit_mac(&mut self, mac: &ast::Mac) {
-            let span = mac.span.with_ctxt(self.span.ctxt());
-            self.cx.span_err(span, "`derive` cannot be used on items with type macros");
+            self.cx.span_err(mac.span(), "`derive` cannot be used on items with type macros");
         }
     }
 
-    let mut visitor = Visitor {
-        ty_param_names,
-        types: Vec::new(),
-        span,
-        cx,
-    };
-
+    let mut visitor = Visitor { cx, ty_param_names, types: Vec::new() };
     visit::Visitor::visit_ty(&mut visitor, ty);
 
     visitor.types
@@ -605,7 +596,7 @@ impl<'a> TraitDef<'a> {
                     .collect();
 
                 for field_ty in field_tys {
-                    let tys = find_type_parameters(&field_ty, &ty_param_names, self.span, cx);
+                    let tys = find_type_parameters(&field_ty, &ty_param_names, cx);
 
                     for ty in tys {
                         // if we have already handled this type, skip it