Introduce `InvisibleOrigin` on invisible delimiters.

It's not used meaningfully yet, but will be needed to get rid of
interpolated tokens.

1 files changed, 108 insertions, 6 deletions

diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index 3b9edef0615..592b56cc08f 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -42,11 +42,86 @@ pub enum BinOpToken {
     Shr,
 }
 
+// This type must not implement `Hash` due to the unusual `PartialEq` impl below.
+#[derive(Copy, Clone, Debug, Encodable, Decodable, HashStable_Generic)]
+pub enum InvisibleOrigin {
+    // From the expansion of a metavariable in a declarative macro.
+    MetaVar(MetaVarKind),
+
+    // Converted from `proc_macro::Delimiter` in
+    // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
+    ProcMacro,
+
+    // Converted from `TokenKind::Interpolated` in
+    // `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
+    FlattenToken,
+}
+
+impl PartialEq for InvisibleOrigin {
+    #[inline]
+    fn eq(&self, _other: &InvisibleOrigin) -> bool {
+        // When we had AST-based nonterminals we couldn't compare them, and the
+        // old `Nonterminal` type had an `eq` that always returned false,
+        // resulting in this restriction:
+        // https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment
+        // This `eq` emulates that behaviour. We could consider lifting this
+        // restriction now but there are still cases involving invisible
+        // delimiters that make it harder than it first appears.
+        false
+    }
+}
+
+/// Annoyingly similar to `NonterminalKind`, but the slight differences are important.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
+pub enum MetaVarKind {
+    Item,
+    Block,
+    Stmt,
+    Pat(NtPatKind),
+    Expr {
+        kind: NtExprKind,
+        // This field is needed for `Token::can_begin_literal_maybe_minus`.
+        can_begin_literal_maybe_minus: bool,
+        // This field is needed for `Token::can_begin_string_literal`.
+        can_begin_string_literal: bool,
+    },
+    Ty,
+    Ident,
+    Lifetime,
+    Literal,
+    Meta,
+    Path,
+    Vis,
+    TT,
+}
+
+impl fmt::Display for MetaVarKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let sym = match self {
+            MetaVarKind::Item => sym::item,
+            MetaVarKind::Block => sym::block,
+            MetaVarKind::Stmt => sym::stmt,
+            MetaVarKind::Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
+            MetaVarKind::Pat(PatParam { inferred: false }) => sym::pat_param,
+            MetaVarKind::Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
+            MetaVarKind::Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
+            MetaVarKind::Ty => sym::ty,
+            MetaVarKind::Ident => sym::ident,
+            MetaVarKind::Lifetime => sym::lifetime,
+            MetaVarKind::Literal => sym::literal,
+            MetaVarKind::Meta => sym::meta,
+            MetaVarKind::Path => sym::path,
+            MetaVarKind::Vis => sym::vis,
+            MetaVarKind::TT => sym::tt,
+        };
+        write!(f, "{sym}")
+    }
+}
+
 /// Describes how a sequence of token trees is delimited.
 /// Cannot use `proc_macro::Delimiter` directly because this
 /// structure should implement some additional traits.
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-#[derive(Encodable, Decodable, Hash, HashStable_Generic)]
+#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum Delimiter {
     /// `( ... )`
     Parenthesis,
@@ -59,7 +134,34 @@ pub enum Delimiter {
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
     /// Invisible delimiters might not survive roundtrip of a token stream through a string.
-    Invisible,
+    Invisible(InvisibleOrigin),
+}
+
+impl Delimiter {
+    // Should the parser skip these delimiters? Only happens for certain kinds
+    // of invisible delimiters. Ideally this function will eventually disappear
+    // and no invisible delimiters will be skipped.
+    #[inline]
+    pub fn skip(&self) -> bool {
+        match self {
+            Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
+            Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false,
+            Delimiter::Invisible(InvisibleOrigin::FlattenToken | InvisibleOrigin::ProcMacro) => {
+                true
+            }
+        }
+    }
+
+    // This exists because `InvisibleOrigin`s should be compared. It is only used for assertions.
+    pub fn eq_ignoring_invisible_origin(&self, other: &Delimiter) -> bool {
+        match (self, other) {
+            (Delimiter::Parenthesis, Delimiter::Parenthesis) => true,
+            (Delimiter::Brace, Delimiter::Brace) => true,
+            (Delimiter::Bracket, Delimiter::Bracket) => true,
+            (Delimiter::Invisible(_), Delimiter::Invisible(_)) => true,
+            _ => false,
+        }
+    }
 }
 
 // Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -896,7 +998,7 @@ impl PartialEq<TokenKind> for Token {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NtPatKind {
     // Matches or-patterns. Was written using `pat` in edition 2021 or later.
     PatWithOr,
@@ -906,7 +1008,7 @@ pub enum NtPatKind {
     PatParam { inferred: bool },
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NtExprKind {
     // Matches expressions using the post-edition 2024. Was written using
     // `expr` in edition 2024 or later.
@@ -933,7 +1035,7 @@ pub enum Nonterminal {
     NtVis(P<ast::Visibility>),
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NonterminalKind {
     Item,
     Block,