From 5326361fc00544f7ba77b250e9646cf05cd07c43 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 1 Sep 2020 11:24:52 +0200 Subject: Remove trivia tokens --- compiler/rustc_ast/src/token.rs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) (limited to 'compiler/rustc_ast/src') diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 4a8bf6b4f19..c6cc890b47f 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -251,17 +251,6 @@ pub enum TokenKind { /// similarly to symbols in string literal tokens. DocComment(CommentKind, ast::AttrStyle, Symbol), - // Junk. These carry no data because we don't really care about the data - // they *would* carry, and don't really want to allocate a new ident for - // them. Instead, users could extract that from the associated span. - /// Whitespace. - Whitespace, - /// A comment. - Comment, - Shebang(Symbol), - /// A completely invalid token which should be skipped. - Unknown(Symbol), - Eof, } @@ -331,7 +320,7 @@ impl Token { /// Some token that will be thrown away later. pub fn dummy() -> Self { - Token::new(TokenKind::Whitespace, DUMMY_SP) + Token::new(TokenKind::Question, DUMMY_SP) } /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary. @@ -360,7 +349,7 @@ impl Token { pub fn is_op(&self) -> bool { match self.kind { OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false, + | Lifetime(..) | Interpolated(..) | Eof => false, _ => true, } } @@ -676,8 +665,7 @@ impl Token { Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment - | Shebang(..) | Unknown(..) | Eof => return None, + | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, }; Some(Token::new(kind, self.span.to(joint.span))) -- cgit 1.4.1-3-g733a5