Auto merge of #139897 - nnethercote:rm-OpenDelim-CloseDelim, r=petrochenkov

Remove `token::{Open,Close}Delim`

By replacing them with `{Open,Close}{Param,Brace,Bracket,Invisible}`.

PR #137902 made `ast::TokenKind` more like `lexer::TokenKind` by
replacing the compound `BinOp{,Eq}(BinOpToken)` variants with fieldless
variants `Plus`, `Minus`, `Star`, etc. This commit does a similar thing
with delimiters. It also makes `ast::TokenKind` more similar to
`parser::TokenType`.

This requires a few new methods:
- `TokenKind::is_{,open_,close_}delim()` replace various kinds of
  pattern matches.
- `Delimiter::as_{open,close}_token_kind` are used to convert
  `Delimiter` values to `TokenKind`.

Despite these additions, it's a net reduction in lines of code. This is
because e.g. `token::OpenParen` is so much shorter than
`token::OpenDelim(Delimiter::Parenthesis)` that many multi-line forms
reduce to single line forms. And many places where the number of lines
doesn't change are still easier to read, just because the names are
shorter, e.g.:
```
-   } else if self.token != token::CloseDelim(Delimiter::Brace) {
+   } else if self.token != token::CloseBrace {
```

r? `@petrochenkov`

7 files changed, 25 insertions, 31 deletions

diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index c70e259b2cd..d2e45d717d9 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -237,10 +237,7 @@ impl<'a> StripUnconfigured<'a> {
                     inner = self.configure_tokens(&inner);
                     Some(AttrTokenTree::Delimited(sp, spacing, delim, inner))
                 }
-                AttrTokenTree::Token(
-                    Token { kind: TokenKind::OpenDelim(_) | TokenKind::CloseDelim(_), .. },
-                    _,
-                ) => {
+                AttrTokenTree::Token(Token { kind, .. }, _) if kind.is_delim() => {
                     panic!("Should be `AttrTokenTree::Delimited`, not delim tokens: {:?}", tree);
                 }
                 AttrTokenTree::Token(token, spacing) => Some(AttrTokenTree::Token(token, spacing)),
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index 1e26d668194..1f430b0018f 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -7,13 +7,12 @@ use std::{iter, mem};
 use rustc_ast as ast;
 use rustc_ast::mut_visit::*;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Delimiter};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{self, AssocCtxt, Visitor, VisitorResult, try_visit, walk_list};
 use rustc_ast::{
     AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, ForeignItemKind,
     HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner, MetaItemKind, ModKind,
-    NodeId, PatKind, StmtKind, TyKind,
+    NodeId, PatKind, StmtKind, TyKind, token,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
@@ -1004,7 +1003,7 @@ pub fn parse_ast_fragment<'a>(
         AstFragmentKind::Stmts => {
             let mut stmts = SmallVec::new();
             // Won't make progress on a `}`.
-            while this.token != token::Eof && this.token != token::CloseDelim(Delimiter::Brace) {
+            while this.token != token::Eof && this.token != token::CloseBrace {
                 if let Some(stmt) = this.parse_full_stmt(AttemptLocalParseRecovery::Yes)? {
                     stmts.push(stmt);
                 }
diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs
index b663e959744..698492f42e2 100644
--- a/compiler/rustc_expand/src/mbe/diagnostics.rs
+++ b/compiler/rustc_expand/src/mbe/diagnostics.rs
@@ -1,6 +1,6 @@
 use std::borrow::Cow;
 
-use rustc_ast::token::{self, Delimiter, Token, TokenKind};
+use rustc_ast::token::{self, Token};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage};
 use rustc_macros::Subdiagnostic;
@@ -66,8 +66,8 @@ pub(super) fn failed_to_match_macro(
     }
 
     if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
-        && (matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
-            || matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))))
+        && (matches!(expected_token.kind, token::OpenInvisible(_))
+            || matches!(token.kind, token::OpenInvisible(_)))
     {
         err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
         err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");
diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs
index 0065f83eb4e..c78beb40688 100644
--- a/compiler/rustc_expand/src/mbe/macro_parser.rs
+++ b/compiler/rustc_expand/src/mbe/macro_parser.rs
@@ -182,8 +182,8 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
                     locs.push(MatcherLoc::Token { token: *token });
                 }
                 TokenTree::Delimited(span, _, delimited) => {
-                    let open_token = Token::new(token::OpenDelim(delimited.delim), span.open);
-                    let close_token = Token::new(token::CloseDelim(delimited.delim), span.close);
+                    let open_token = Token::new(delimited.delim.as_open_token_kind(), span.open);
+                    let close_token = Token::new(delimited.delim.as_close_token_kind(), span.close);
 
                     locs.push(MatcherLoc::Delimited);
                     locs.push(MatcherLoc::Token { token: open_token });
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index c138b090877..93604a149f1 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -6,7 +6,7 @@ use std::{mem, slice};
 use ast::token::IdentIsRaw;
 use rustc_ast::token::NtPatKind::*;
 use rustc_ast::token::TokenKind::*;
-use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind};
+use rustc_ast::token::{self, NonterminalKind, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream};
 use rustc_ast::{self as ast, DUMMY_NODE_ID, NodeId};
 use rustc_ast_pretty::pprust;
@@ -784,7 +784,7 @@ impl<'tt> FirstSets<'tt> {
                     TokenTree::Delimited(span, _, delimited) => {
                         build_recur(sets, &delimited.tts);
                         first.replace_with(TtHandle::from_token_kind(
-                            token::OpenDelim(delimited.delim),
+                            delimited.delim.as_open_token_kind(),
                             span.open,
                         ));
                     }
@@ -852,7 +852,7 @@ impl<'tt> FirstSets<'tt> {
                 }
                 TokenTree::Delimited(span, _, delimited) => {
                     first.add_one(TtHandle::from_token_kind(
-                        token::OpenDelim(delimited.delim),
+                        delimited.delim.as_open_token_kind(),
                         span.open,
                     ));
                     return first;
@@ -1099,7 +1099,7 @@ fn check_matcher_core<'tt>(
             }
             TokenTree::Delimited(span, _, d) => {
                 let my_suffix = TokenSet::singleton(TtHandle::from_token_kind(
-                    token::CloseDelim(d.delim),
+                    d.delim.as_close_token_kind(),
                     span.close,
                 ));
                 check_matcher_core(sess, node_id, first_sets, &d.tts, &my_suffix)?;
@@ -1299,7 +1299,9 @@ enum IsInFollow {
 fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
     use mbe::TokenTree;
 
-    if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok {
+    if let TokenTree::Token(Token { kind, .. }) = tok
+        && kind.close_delim().is_some()
+    {
         // closing a token tree can never be matched by any fragment;
         // iow, we always require that `(` and `)` match, etc.
         IsInFollow::Yes
@@ -1358,16 +1360,8 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                 ];
                 match tok {
                     TokenTree::Token(token) => match token.kind {
-                        OpenDelim(Delimiter::Brace)
-                        | OpenDelim(Delimiter::Bracket)
-                        | Comma
-                        | FatArrow
-                        | Colon
-                        | Eq
-                        | Gt
-                        | Shr
-                        | Semi
-                        | Or => IsInFollow::Yes,
+                        OpenBrace | OpenBracket | Comma | FatArrow | Colon | Eq | Gt | Shr
+                        | Semi | Or => IsInFollow::Yes,
                         Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
                             IsInFollow::Yes
                         }
diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs
index 3f037259956..0c2362f23bc 100644
--- a/compiler/rustc_expand/src/mbe/quoted.rs
+++ b/compiler/rustc_expand/src/mbe/quoted.rs
@@ -181,7 +181,10 @@ fn parse_tree<'a>(
                         if delim != Delimiter::Parenthesis {
                             span_dollar_dollar_or_metavar_in_the_lhs_err(
                                 sess,
-                                &Token { kind: token::OpenDelim(delim), span: delim_span.entire() },
+                                &Token {
+                                    kind: delim.as_open_token_kind(),
+                                    span: delim_span.entire(),
+                                },
                             );
                         }
                     } else {
@@ -217,7 +220,8 @@ fn parse_tree<'a>(
                             }
                             Delimiter::Parenthesis => {}
                             _ => {
-                                let token = pprust::token_kind_to_string(&token::OpenDelim(delim));
+                                let token =
+                                    pprust::token_kind_to_string(&delim.as_open_token_kind());
                                 sess.dcx().emit_err(errors::ExpectedParenOrBrace {
                                     span: delim_span.entire(),
                                     token,
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index 4edaf68c89a..f00201ad202 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -308,8 +308,8 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     }));
                 }
 
-                OpenDelim(..) | CloseDelim(..) => unreachable!(),
-                Eof => unreachable!(),
+                OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
+                | OpenInvisible(_) | CloseInvisible(_) | Eof => unreachable!(),
             }
         }
         trees