Auto merge of #17535 - Veykril:macro-def-syn, r=Veykril

fix: Fix up the syntax tree for macro 2.0

Fixes https://github.com/rust-lang/rust-analyzer/issues/10266

This change is trivial now that we don't have a token map anymore

9 files changed, 218 insertions, 161 deletions

diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs b/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs
index 7c3bf995b12..29408902f16 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs
@@ -172,15 +172,30 @@ impl DeclarativeMacroExpander {
             ),
             ast::Macro::MacroDef(macro_def) => (
                 match macro_def.body() {
-                    Some(arg) => {
-                        let tt = mbe::syntax_node_to_token_tree(
-                            arg.syntax(),
+                    Some(body) => {
+                        let span =
+                            map.span_for_range(macro_def.macro_token().unwrap().text_range());
+                        let args = macro_def.args().map(|args| {
+                            mbe::syntax_node_to_token_tree(
+                                args.syntax(),
+                                map.as_ref(),
+                                span,
+                                DocCommentDesugarMode::Mbe,
+                            )
+                        });
+                        let body = mbe::syntax_node_to_token_tree(
+                            body.syntax(),
                             map.as_ref(),
-                            map.span_for_range(macro_def.macro_token().unwrap().text_range()),
+                            span,
                             DocCommentDesugarMode::Mbe,
                         );
 
-                        mbe::DeclarativeMacro::parse_macro2(&tt, edition, new_meta_vars)
+                        mbe::DeclarativeMacro::parse_macro2(
+                            args.as_ref(),
+                            &body,
+                            edition,
+                            new_meta_vars,
+                        )
                     }
                     None => mbe::DeclarativeMacro::from_err(mbe::ParseError::Expected(
                         "expected a token tree".into(),
diff --git a/src/tools/rust-analyzer/crates/mbe/src/lib.rs b/src/tools/rust-analyzer/crates/mbe/src/lib.rs
index b2fd91f6ced..b06c6cee12d 100644
--- a/src/tools/rust-analyzer/crates/mbe/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/mbe/src/lib.rs
@@ -156,7 +156,7 @@ impl DeclarativeMacro {
         let mut err = None;
 
         while src.len() > 0 {
-            let rule = match Rule::parse(edition, &mut src, true, new_meta_vars) {
+            let rule = match Rule::parse(edition, &mut src, new_meta_vars) {
                 Ok(it) => it,
                 Err(e) => {
                     err = Some(Box::new(e));
@@ -184,19 +184,34 @@ impl DeclarativeMacro {
 
     /// The new, unstable `macro m {}` flavor.
     pub fn parse_macro2(
-        tt: &tt::Subtree<Span>,
+        args: Option<&tt::Subtree<Span>>,
+        body: &tt::Subtree<Span>,
         edition: impl Copy + Fn(SyntaxContextId) -> Edition,
         // FIXME: Remove this once we drop support for rust 1.76 (defaults to true then)
         new_meta_vars: bool,
     ) -> DeclarativeMacro {
-        let mut src = TtIter::new(tt);
         let mut rules = Vec::new();
         let mut err = None;
 
-        if tt::DelimiterKind::Brace == tt.delimiter.kind {
+        if let Some(args) = args {
+            cov_mark::hit!(parse_macro_def_simple);
+
+            let rule = (|| {
+                let lhs = MetaTemplate::parse_pattern(edition, args)?;
+                let rhs = MetaTemplate::parse_template(edition, body, new_meta_vars)?;
+
+                Ok(crate::Rule { lhs, rhs })
+            })();
+
+            match rule {
+                Ok(rule) => rules.push(rule),
+                Err(e) => err = Some(Box::new(e)),
+            }
+        } else {
             cov_mark::hit!(parse_macro_def_rules);
+            let mut src = TtIter::new(body);
             while src.len() > 0 {
-                let rule = match Rule::parse(edition, &mut src, true, new_meta_vars) {
+                let rule = match Rule::parse(edition, &mut src, new_meta_vars) {
                     Ok(it) => it,
                     Err(e) => {
                         err = Some(Box::new(e));
@@ -213,19 +228,6 @@ impl DeclarativeMacro {
                     break;
                 }
             }
-        } else {
-            cov_mark::hit!(parse_macro_def_simple);
-            match Rule::parse(edition, &mut src, false, new_meta_vars) {
-                Ok(rule) => {
-                    if src.len() != 0 {
-                        err = Some(Box::new(ParseError::expected("remaining tokens in macro def")));
-                    }
-                    rules.push(rule);
-                }
-                Err(e) => {
-                    err = Some(Box::new(e));
-                }
-            }
         }
 
         for Rule { lhs, .. } in &rules {
@@ -262,14 +264,11 @@ impl Rule {
     fn parse(
         edition: impl Copy + Fn(SyntaxContextId) -> Edition,
         src: &mut TtIter<'_, Span>,
-        expect_arrow: bool,
         new_meta_vars: bool,
     ) -> Result<Self, ParseError> {
         let lhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?;
-        if expect_arrow {
-            src.expect_char('=').map_err(|()| ParseError::expected("expected `=`"))?;
-            src.expect_char('>').map_err(|()| ParseError::expected("expected `>`"))?;
-        }
+        src.expect_char('=').map_err(|()| ParseError::expected("expected `=`"))?;
+        src.expect_char('>').map_err(|()| ParseError::expected("expected `>`"))?;
         let rhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?;
 
         let lhs = MetaTemplate::parse_pattern(edition, lhs)?;
diff --git a/src/tools/rust-analyzer/crates/parser/src/grammar/items.rs b/src/tools/rust-analyzer/crates/parser/src/grammar/items.rs
index 25c00ccf5f3..99bbf47654b 100644
--- a/src/tools/rust-analyzer/crates/parser/src/grammar/items.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/grammar/items.rs
@@ -372,13 +372,11 @@ fn macro_def(p: &mut Parser<'_>, m: Marker) {
         // macro m { ($i:ident) => {} }
         token_tree(p);
     } else if p.at(T!['(']) {
-        let m = p.start();
         token_tree(p);
         match p.current() {
             T!['{'] | T!['['] | T!['('] => token_tree(p),
             _ => p.error("expected `{`, `[`, `(`"),
         }
-        m.complete(p, TOKEN_TREE);
     } else {
         p.error("unmatched `(`");
     }
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0147_macro_def.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0147_macro_def.rast
index 01de13a907d..f73229b2e30 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0147_macro_def.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0147_macro_def.rast
@@ -5,15 +5,14 @@ SOURCE_FILE
     NAME
       IDENT "m"
     TOKEN_TREE
-      TOKEN_TREE
-        L_PAREN "("
-        DOLLAR "$"
-        IDENT "i"
-        COLON ":"
-        IDENT "ident"
-        R_PAREN ")"
-      WHITESPACE " "
-      TOKEN_TREE
-        L_CURLY "{"
-        R_CURLY "}"
+      L_PAREN "("
+      DOLLAR "$"
+      IDENT "i"
+      COLON ":"
+      IDENT "ident"
+      R_PAREN ")"
+    WHITESPACE " "
+    TOKEN_TREE
+      L_CURLY "{"
+      R_CURLY "}"
   WHITESPACE "\n"
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0012_visibility.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0012_visibility.rast
index a95bc23016b..3d9322947b3 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0012_visibility.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0012_visibility.rast
@@ -39,16 +39,15 @@ SOURCE_FILE
     NAME
       IDENT "m"
     TOKEN_TREE
-      TOKEN_TREE
-        L_PAREN "("
-        DOLLAR "$"
-        COLON ":"
-        IDENT "ident"
-        R_PAREN ")"
-      WHITESPACE " "
-      TOKEN_TREE
-        L_CURLY "{"
-        R_CURLY "}"
+      L_PAREN "("
+      DOLLAR "$"
+      COLON ":"
+      IDENT "ident"
+      R_PAREN ")"
+    WHITESPACE " "
+    TOKEN_TREE
+      L_CURLY "{"
+      R_CURLY "}"
   WHITESPACE "\n"
   FN
     VISIBILITY
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0062_macro_2.0.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0062_macro_2.0.rast
index 3915ed75064..1415a866b69 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0062_macro_2.0.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0062_macro_2.0.rast
@@ -5,51 +5,50 @@ SOURCE_FILE
     NAME
       IDENT "parse_use_trees"
     TOKEN_TREE
+      L_PAREN "("
+      DOLLAR "$"
       TOKEN_TREE
         L_PAREN "("
         DOLLAR "$"
-        TOKEN_TREE
-          L_PAREN "("
-          DOLLAR "$"
-          IDENT "s"
-          COLON ":"
-          IDENT "expr"
-          R_PAREN ")"
+        IDENT "s"
+        COLON ":"
+        IDENT "expr"
+        R_PAREN ")"
+      COMMA ","
+      STAR "*"
+      WHITESPACE " "
+      DOLLAR "$"
+      TOKEN_TREE
+        L_PAREN "("
         COMMA ","
-        STAR "*"
-        WHITESPACE " "
+        R_PAREN ")"
+      STAR "*"
+      R_PAREN ")"
+    WHITESPACE " "
+    TOKEN_TREE
+      L_CURLY "{"
+      WHITESPACE "\n    "
+      IDENT "vec"
+      BANG "!"
+      TOKEN_TREE
+        L_BRACK "["
+        WHITESPACE "\n        "
         DOLLAR "$"
         TOKEN_TREE
           L_PAREN "("
+          IDENT "parse_use_tree"
+          TOKEN_TREE
+            L_PAREN "("
+            DOLLAR "$"
+            IDENT "s"
+            R_PAREN ")"
           COMMA ","
           R_PAREN ")"
         STAR "*"
-        R_PAREN ")"
-      WHITESPACE " "
-      TOKEN_TREE
-        L_CURLY "{"
         WHITESPACE "\n    "
-        IDENT "vec"
-        BANG "!"
-        TOKEN_TREE
-          L_BRACK "["
-          WHITESPACE "\n        "
-          DOLLAR "$"
-          TOKEN_TREE
-            L_PAREN "("
-            IDENT "parse_use_tree"
-            TOKEN_TREE
-              L_PAREN "("
-              DOLLAR "$"
-              IDENT "s"
-              R_PAREN ")"
-            COMMA ","
-            R_PAREN ")"
-          STAR "*"
-          WHITESPACE "\n    "
-          R_BRACK "]"
-        WHITESPACE "\n"
-        R_CURLY "}"
+        R_BRACK "]"
+      WHITESPACE "\n"
+      R_CURLY "}"
   WHITESPACE "\n\n"
   FN
     ATTR
@@ -80,79 +79,62 @@ SOURCE_FILE
           NAME
             IDENT "test_merge"
           TOKEN_TREE
+            L_PAREN "("
             TOKEN_TREE
-              L_PAREN "("
+              L_BRACK "["
+              DOLLAR "$"
               TOKEN_TREE
-                L_BRACK "["
-                DOLLAR "$"
-                TOKEN_TREE
-                  L_PAREN "("
-                  DOLLAR "$"
-                  IDENT "input"
-                  COLON ":"
-                  IDENT "expr"
-                  R_PAREN ")"
-                COMMA ","
-                STAR "*"
-                WHITESPACE " "
+                L_PAREN "("
                 DOLLAR "$"
-                TOKEN_TREE
-                  L_PAREN "("
-                  COMMA ","
-                  R_PAREN ")"
-                STAR "*"
-                R_BRACK "]"
+                IDENT "input"
+                COLON ":"
+                IDENT "expr"
+                R_PAREN ")"
               COMMA ","
+              STAR "*"
               WHITESPACE " "
+              DOLLAR "$"
               TOKEN_TREE
-                L_BRACK "["
-                DOLLAR "$"
-                TOKEN_TREE
-                  L_PAREN "("
-                  DOLLAR "$"
-                  IDENT "output"
-                  COLON ":"
-                  IDENT "expr"
-                  R_PAREN ")"
+                L_PAREN "("
                 COMMA ","
-                STAR "*"
-                WHITESPACE " "
-                DOLLAR "$"
-                TOKEN_TREE
-                  L_PAREN "("
-                  COMMA ","
-                  R_PAREN ")"
-                STAR "*"
-                R_BRACK "]"
-              R_PAREN ")"
+                R_PAREN ")"
+              STAR "*"
+              R_BRACK "]"
+            COMMA ","
             WHITESPACE " "
             TOKEN_TREE
-              L_CURLY "{"
-              WHITESPACE "\n        "
-              IDENT "assert_eq"
-              BANG "!"
+              L_BRACK "["
+              DOLLAR "$"
+              TOKEN_TREE
+                L_PAREN "("
+                DOLLAR "$"
+                IDENT "output"
+                COLON ":"
+                IDENT "expr"
+                R_PAREN ")"
+              COMMA ","
+              STAR "*"
+              WHITESPACE " "
+              DOLLAR "$"
               TOKEN_TREE
                 L_PAREN "("
-                WHITESPACE "\n            "
-                IDENT "merge_use_trees"
-                TOKEN_TREE
-                  L_PAREN "("
-                  IDENT "parse_use_trees"
-                  BANG "!"
-                  TOKEN_TREE
-                    L_PAREN "("
-                    DOLLAR "$"
-                    TOKEN_TREE
-                      L_PAREN "("
-                      DOLLAR "$"
-                      IDENT "input"
-                      COMMA ","
-                      R_PAREN ")"
-                    STAR "*"
-                    R_PAREN ")"
-                  R_PAREN ")"
                 COMMA ","
-                WHITESPACE "\n            "
+                R_PAREN ")"
+              STAR "*"
+              R_BRACK "]"
+            R_PAREN ")"
+          WHITESPACE " "
+          TOKEN_TREE
+            L_CURLY "{"
+            WHITESPACE "\n        "
+            IDENT "assert_eq"
+            BANG "!"
+            TOKEN_TREE
+              L_PAREN "("
+              WHITESPACE "\n            "
+              IDENT "merge_use_trees"
+              TOKEN_TREE
+                L_PAREN "("
                 IDENT "parse_use_trees"
                 BANG "!"
                 TOKEN_TREE
@@ -161,17 +143,33 @@ SOURCE_FILE
                   TOKEN_TREE
                     L_PAREN "("
                     DOLLAR "$"
-                    IDENT "output"
+                    IDENT "input"
                     COMMA ","
                     R_PAREN ")"
                   STAR "*"
                   R_PAREN ")"
-                COMMA ","
-                WHITESPACE "\n        "
                 R_PAREN ")"
-              SEMICOLON ";"
-              WHITESPACE "\n    "
-              R_CURLY "}"
+              COMMA ","
+              WHITESPACE "\n            "
+              IDENT "parse_use_trees"
+              BANG "!"
+              TOKEN_TREE
+                L_PAREN "("
+                DOLLAR "$"
+                TOKEN_TREE
+                  L_PAREN "("
+                  DOLLAR "$"
+                  IDENT "output"
+                  COMMA ","
+                  R_PAREN ")"
+                STAR "*"
+                R_PAREN ")"
+              COMMA ","
+              WHITESPACE "\n        "
+              R_PAREN ")"
+            SEMICOLON ";"
+            WHITESPACE "\n    "
+            R_CURLY "}"
         WHITESPACE "\n"
         R_CURLY "}"
   WHITESPACE "\n"
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/expr_ext.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/expr_ext.rs
index 28a9dadacef..b0ee9dfd507 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/expr_ext.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/expr_ext.rs
@@ -6,7 +6,8 @@ use crate::{
     ast::{
         self,
         operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp},
-        support, AstChildren, AstNode,
+        support, ArgList, AstChildren, AstNode, BlockExpr, ClosureExpr, Const, Expr, Fn,
+        FormatArgsArg, FormatArgsExpr, MacroDef, Static, TokenTree,
     },
     AstToken,
     SyntaxKind::*,
@@ -435,3 +436,57 @@ impl AstNode for CallableExpr {
         }
     }
 }
+
+impl MacroDef {
+    fn tts(&self) -> (Option<ast::TokenTree>, Option<ast::TokenTree>) {
+        let mut types = support::children(self.syntax());
+        let first = types.next();
+        let second = types.next();
+        (first, second)
+    }
+
+    pub fn args(&self) -> Option<TokenTree> {
+        match self.tts() {
+            (Some(args), Some(_)) => Some(args),
+            _ => None,
+        }
+    }
+
+    pub fn body(&self) -> Option<TokenTree> {
+        match self.tts() {
+            (Some(body), None) | (_, Some(body)) => Some(body),
+            _ => None,
+        }
+    }
+}
+
+impl ClosureExpr {
+    pub fn body(&self) -> Option<Expr> {
+        support::child(&self.syntax)
+    }
+}
+impl Const {
+    pub fn body(&self) -> Option<Expr> {
+        support::child(&self.syntax)
+    }
+}
+impl Fn {
+    pub fn body(&self) -> Option<BlockExpr> {
+        support::child(&self.syntax)
+    }
+}
+impl Static {
+    pub fn body(&self) -> Option<Expr> {
+        support::child(&self.syntax)
+    }
+}
+impl FormatArgsExpr {
+    pub fn args(&self) -> AstChildren<FormatArgsArg> {
+        support::children(&self.syntax)
+    }
+}
+impl ArgList {
+    pub fn args(&self) -> AstChildren<Expr> {
+        support::children(&self.syntax)
+    }
+}
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs
index 98186c5473d..bae529a2c8e 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs
@@ -20,7 +20,6 @@ pub struct ArgList {
     pub(crate) syntax: SyntaxNode,
 }
 impl ArgList {
-    pub fn args(&self) -> AstChildren<Expr> { support::children(&self.syntax) }
     pub fn l_paren_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T!['(']) }
     pub fn r_paren_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![')']) }
 }
@@ -191,7 +190,6 @@ pub struct ClosureExpr {
 }
 impl ast::HasAttrs for ClosureExpr {}
 impl ClosureExpr {
-    pub fn body(&self) -> Option<Expr> { support::child(&self.syntax) }
     pub fn generic_param_list(&self) -> Option<GenericParamList> { support::child(&self.syntax) }
     pub fn param_list(&self) -> Option<ParamList> { support::child(&self.syntax) }
     pub fn ret_type(&self) -> Option<RetType> { support::child(&self.syntax) }
@@ -211,7 +209,6 @@ impl ast::HasDocComments for Const {}
 impl ast::HasName for Const {}
 impl ast::HasVisibility for Const {}
 impl Const {
-    pub fn body(&self) -> Option<Expr> { support::child(&self.syntax) }
     pub fn ty(&self) -> Option<Type> { support::child(&self.syntax) }
     pub fn colon_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![:]) }
     pub fn semicolon_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![;]) }
@@ -356,7 +353,6 @@ impl ast::HasName for Fn {}
 impl ast::HasVisibility for Fn {}
 impl Fn {
     pub fn abi(&self) -> Option<Abi> { support::child(&self.syntax) }
-    pub fn body(&self) -> Option<BlockExpr> { support::child(&self.syntax) }
     pub fn param_list(&self) -> Option<ParamList> { support::child(&self.syntax) }
     pub fn ret_type(&self) -> Option<RetType> { support::child(&self.syntax) }
     pub fn semicolon_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![;]) }
@@ -418,7 +414,6 @@ pub struct FormatArgsExpr {
 }
 impl ast::HasAttrs for FormatArgsExpr {}
 impl FormatArgsExpr {
-    pub fn args(&self) -> AstChildren<FormatArgsArg> { support::children(&self.syntax) }
     pub fn template(&self) -> Option<Expr> { support::child(&self.syntax) }
     pub fn pound_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![#]) }
     pub fn l_paren_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T!['(']) }
@@ -652,8 +647,6 @@ impl ast::HasDocComments for MacroDef {}
 impl ast::HasName for MacroDef {}
 impl ast::HasVisibility for MacroDef {}
 impl MacroDef {
-    pub fn args(&self) -> Option<TokenTree> { support::child(&self.syntax) }
-    pub fn body(&self) -> Option<TokenTree> { support::child(&self.syntax) }
     pub fn macro_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![macro]) }
 }
 
@@ -1224,7 +1217,6 @@ impl ast::HasDocComments for Static {}
 impl ast::HasName for Static {}
 impl ast::HasVisibility for Static {}
 impl Static {
-    pub fn body(&self) -> Option<Expr> { support::child(&self.syntax) }
     pub fn ty(&self) -> Option<Type> { support::child(&self.syntax) }
     pub fn colon_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![:]) }
     pub fn semicolon_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![;]) }
diff --git a/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs b/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs
index cc2fadc9750..0fc97ad6541 100644
--- a/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs
+++ b/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs
@@ -692,6 +692,8 @@ fn lower_rule(acc: &mut Vec<Field>, grammar: &Grammar, label: Option<&String>, r
                     | "self_ty"
                     | "iterable"
                     | "condition"
+                    | "args"
+                    | "body"
             );
             if manually_implemented {
                 return;