diff options
Diffstat (limited to 'compiler/rustc_ast/src/token.rs')
| -rw-r--r-- | compiler/rustc_ast/src/token.rs | 131 |
1 files changed, 103 insertions, 28 deletions
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 055481f5d87..54781e8235e 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -32,6 +32,18 @@ pub enum InvisibleOrigin { ProcMacro, } +impl InvisibleOrigin { + // Should the parser skip these invisible delimiters? Ideally this function + // will eventually disappear and no invisible delimiters will be skipped. + #[inline] + pub fn skip(&self) -> bool { + match self { + InvisibleOrigin::MetaVar(_) => false, + InvisibleOrigin::ProcMacro => true, + } + } +} + impl PartialEq for InvisibleOrigin { #[inline] fn eq(&self, _other: &InvisibleOrigin) -> bool { @@ -125,8 +137,7 @@ impl Delimiter { pub fn skip(&self) -> bool { match self { Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false, - Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false, - Delimiter::Invisible(InvisibleOrigin::ProcMacro) => true, + Delimiter::Invisible(origin) => origin.skip(), } } @@ -140,6 +151,24 @@ impl Delimiter { _ => false, } } + + pub fn as_open_token_kind(&self) -> TokenKind { + match *self { + Delimiter::Parenthesis => OpenParen, + Delimiter::Brace => OpenBrace, + Delimiter::Bracket => OpenBracket, + Delimiter::Invisible(origin) => OpenInvisible(origin), + } + } + + pub fn as_close_token_kind(&self) -> TokenKind { + match *self { + Delimiter::Parenthesis => CloseParen, + Delimiter::Brace => CloseBrace, + Delimiter::Bracket => CloseBracket, + Delimiter::Invisible(origin) => CloseInvisible(origin), + } + } } // Note that the suffix is *not* considered when deciding the `LitKind` in this @@ -194,9 +223,9 @@ impl Lit { match token.uninterpolate().kind { Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)), Literal(token_lit) => Some(token_lit), - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Literal | MetaVarKind::Expr { .. }, - ))) => { + )) => { // Unreachable with the current test suite. panic!("from_token metavar"); } @@ -426,10 +455,22 @@ pub enum TokenKind { Question, /// Used by proc macros for representing lifetimes, not generated by lexer right now. SingleQuote, - /// An opening delimiter (e.g., `{`). - OpenDelim(Delimiter), - /// A closing delimiter (e.g., `}`). - CloseDelim(Delimiter), + /// `(` + OpenParen, + /// `)` + CloseParen, + /// `{` + OpenBrace, + /// `}` + CloseBrace, + /// `[` + OpenBracket, + /// `]` + CloseBracket, + /// Invisible opening delimiter, produced by a macro. + OpenInvisible(InvisibleOrigin), + /// Invisible closing delimiter, produced by a macro. + CloseInvisible(InvisibleOrigin), /* Literals */ Literal(Lit), @@ -530,6 +571,37 @@ impl TokenKind { pub fn should_end_const_arg(&self) -> bool { matches!(self, Gt | Ge | Shr | ShrEq) } + + pub fn is_delim(&self) -> bool { + self.open_delim().is_some() || self.close_delim().is_some() + } + + pub fn open_delim(&self) -> Option<Delimiter> { + match *self { + OpenParen => Some(Delimiter::Parenthesis), + OpenBrace => Some(Delimiter::Brace), + OpenBracket => Some(Delimiter::Bracket), + OpenInvisible(origin) => Some(Delimiter::Invisible(origin)), + _ => None, + } + } + + pub fn close_delim(&self) -> Option<Delimiter> { + match *self { + CloseParen => Some(Delimiter::Parenthesis), + CloseBrace => Some(Delimiter::Brace), + CloseBracket => Some(Delimiter::Bracket), + CloseInvisible(origin) => Some(Delimiter::Invisible(origin)), + _ => None, + } + } + + pub fn is_close_delim_or_eof(&self) -> bool { + match self { + CloseParen | CloseBrace | CloseBracket | CloseInvisible(_) | Eof => true, + _ => false, + } + } } impl Token { @@ -559,7 +631,8 @@ impl Token { | DotDotDot | DotDotEq | Comma | Semi | Colon | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => true, - OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) + OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket + | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | DocComment(..) | Ident(..) | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false, } } @@ -573,11 +646,12 @@ impl Token { /// **NB**: Take care when modifying this function, since it will change /// the stable set of tokens that are allowed to match an expr nonterminal. pub fn can_begin_expr(&self) -> bool { - use Delimiter::*; match self.uninterpolate().kind { Ident(name, is_raw) => ident_can_begin_expr(name, self.span, is_raw), // value name or keyword - OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block + OpenParen | // tuple + OpenBrace | // block + OpenBracket | // array Literal(..) | // literal Bang | // operator not Minus | // unary minus @@ -591,12 +665,12 @@ impl Token { PathSep | // global path Lifetime(..) | // labeled loop Pound => true, // expression attributes - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Block | MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path - ))) => true, + )) => true, _ => false, } } @@ -608,8 +682,8 @@ impl Token { match &self.uninterpolate().kind { // box, ref, mut, and other identifiers (can stricten) Ident(..) | NtIdent(..) | - OpenDelim(Delimiter::Parenthesis) | // tuple pattern - OpenDelim(Delimiter::Bracket) | // slice pattern + OpenParen | // tuple pattern + OpenBracket | // slice pattern And | // reference Minus | // negative literal AndAnd | // double reference @@ -620,14 +694,14 @@ impl Token { Lt | // path (UFCS constant) Shl => true, // path (double UFCS) Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Meta { .. } | MetaVarKind::Pat(_) | MetaVarKind::Path | MetaVarKind::Ty { .. } - ))) => true, + )) => true, _ => false, } } @@ -637,8 +711,8 @@ impl Token { match self.uninterpolate().kind { Ident(name, is_raw) => ident_can_begin_type(name, self.span, is_raw), // type name or keyword - OpenDelim(Delimiter::Parenthesis) | // tuple - OpenDelim(Delimiter::Bracket) | // array + OpenParen | // tuple + OpenBracket | // array Bang | // never Star | // raw pointer And | // reference @@ -647,10 +721,10 @@ impl Token { Lifetime(..) | // lifetime bound in trait object Lt | Shl | // associated path PathSep => true, // global path - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Ty { .. } | MetaVarKind::Path - ))) => true, + )) => true, // For anonymous structs or unions, which only appear in specific positions // (type of struct fields or union fields), we don't consider them as regular types _ => false, @@ -660,11 +734,11 @@ impl Token { /// Returns `true` if the token can appear at the start of a const param. pub fn can_begin_const_arg(&self) -> bool { match self.kind { - OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true, + OpenBrace | Literal(..) | Minus => true, Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal, - ))) => true, + )) => true, _ => false, } } @@ -711,7 +785,7 @@ impl Token { match self.uninterpolate().kind { Literal(..) | Minus => true, Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind { + OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind { MetaVarKind::Literal => true, MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => { can_begin_literal_maybe_minus @@ -725,7 +799,7 @@ impl Token { pub fn can_begin_string_literal(&self) -> bool { match self.uninterpolate().kind { Literal(..) => true, - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind { + OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind { MetaVarKind::Literal => true, MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal, _ => false, @@ -892,7 +966,7 @@ impl Token { /// from an expanded metavar? pub fn is_metavar_seq(&self) -> Option<MetaVarKind> { match self.kind { - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind), + OpenInvisible(InvisibleOrigin::MetaVar(kind)) => Some(kind), _ => None, } } @@ -970,7 +1044,8 @@ impl Token { Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | PlusEq | MinusEq | StarEq | SlashEq | PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question - | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..) + | OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket + | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | Ident(..) | NtIdent(..) | Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof, _, ) => { |