Overhaul `MacArgs::Eq`.

The value in `MacArgs::Eq` is currently represented as a `Token`.
Because of `TokenKind::Interpolated`, `Token` can be either a token or
an arbitrary AST fragment. In practice, a `MacArgs::Eq` starts out as a
literal or macro call AST fragment, and then is later lowered to a
literal token. But this is very non-obvious. `Token` is a much more
general type than what is needed.

This commit restricts things, by introducing a new type `MacArgsEqKind`
that is either an AST expression (pre-lowering) or an AST literal
(post-lowering). The downside is that the code is a bit more verbose in
a few places. The benefit is that makes it much clearer what the
possibilities are (though also shorter in some other places). Also, it
removes one use of `TokenKind::Interpolated`, taking us a step closer to
removing that variant, which will let us make `Token` impl `Copy` and
remove many "handle Interpolated" code paths in the parser.

Things to note:
- Error messages have improved. Messages like this:
  ```
  unexpected token: `"bug" + "found"`
  ```
  now say "unexpected expression", which makes more sense. Although
  arbitrary expressions can exist within tokens thanks to
  `TokenKind::Interpolated`, that's not obvious to anyone who doesn't
  know compiler internals.
- In `parse_mac_args_common`, we no longer need to collect tokens for
  the value expression.

1 files changed, 19 insertions, 4 deletions

diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index b14367aa1c2..84654a9f737 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -3,14 +3,16 @@
 use crate::ast;
 use crate::ast::{AttrId, AttrItem, AttrKind, AttrStyle, Attribute};
 use crate::ast::{Lit, LitKind};
-use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
+use crate::ast::{MacArgs, MacArgsEq, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
 use crate::ast::{Path, PathSegment};
+use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter, Token};
 use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
 use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing};
 use crate::tokenstream::{LazyTokenStream, TokenStream};
 use crate::util::comments;
 
+use rustc_data_structures::thin_vec::ThinVec;
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_span::source_map::BytePos;
 use rustc_span::symbol::{sym, Ident, Symbol};
@@ -475,7 +477,16 @@ impl MetaItemKind {
     pub fn mac_args(&self, span: Span) -> MacArgs {
         match self {
             MetaItemKind::Word => MacArgs::Empty,
-            MetaItemKind::NameValue(lit) => MacArgs::Eq(span, lit.to_token()),
+            MetaItemKind::NameValue(lit) => {
+                let expr = P(ast::Expr {
+                    id: ast::DUMMY_NODE_ID,
+                    kind: ast::ExprKind::Lit(lit.clone()),
+                    span: lit.span,
+                    attrs: ThinVec::new(),
+                    tokens: None,
+                });
+                MacArgs::Eq(span, MacArgsEq::Ast(expr))
+            }
             MetaItemKind::List(list) => {
                 let mut tts = Vec::new();
                 for (i, item) in list.iter().enumerate() {
@@ -552,12 +563,16 @@ impl MetaItemKind {
 
     fn from_mac_args(args: &MacArgs) -> Option<MetaItemKind> {
         match args {
+            MacArgs::Empty => Some(MetaItemKind::Word),
             MacArgs::Delimited(_, MacDelimiter::Parenthesis, tokens) => {
                 MetaItemKind::list_from_tokens(tokens.clone())
             }
             MacArgs::Delimited(..) => None,
-            MacArgs::Eq(_, token) => Lit::from_token(token).ok().map(MetaItemKind::NameValue),
-            MacArgs::Empty => Some(MetaItemKind::Word),
+            MacArgs::Eq(_, MacArgsEq::Ast(expr)) => match &expr.kind {
+                ast::ExprKind::Lit(lit) => Some(MetaItemKind::NameValue(lit.clone())),
+                _ => None,
+            },
+            MacArgs::Eq(_, MacArgsEq::Hir(lit)) => Some(MetaItemKind::NameValue(lit.clone())),
         }
     }