diff options
| -rw-r--r-- | compiler/rustc_ast/src/token.rs | 18 | ||||
| -rw-r--r-- | compiler/rustc_ast_pretty/src/pprust.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/proc_macro_server.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/lexer/unicode_chars.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/lib.rs | 7 |
5 files changed, 5 insertions, 28 deletions
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 4a8bf6b4f19..c6cc890b47f 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -251,17 +251,6 @@ pub enum TokenKind { /// similarly to symbols in string literal tokens. DocComment(CommentKind, ast::AttrStyle, Symbol), - // Junk. These carry no data because we don't really care about the data - // they *would* carry, and don't really want to allocate a new ident for - // them. Instead, users could extract that from the associated span. - /// Whitespace. - Whitespace, - /// A comment. - Comment, - Shebang(Symbol), - /// A completely invalid token which should be skipped. - Unknown(Symbol), - Eof, } @@ -331,7 +320,7 @@ impl Token { /// Some token that will be thrown away later. pub fn dummy() -> Self { - Token::new(TokenKind::Whitespace, DUMMY_SP) + Token::new(TokenKind::Question, DUMMY_SP) } /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary. @@ -360,7 +349,7 @@ impl Token { pub fn is_op(&self) -> bool { match self.kind { OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false, + | Lifetime(..) | Interpolated(..) | Eof => false, _ => true, } } @@ -676,8 +665,7 @@ impl Token { Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment - | Shebang(..) | Unknown(..) | Eof => return None, + | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, }; Some(Token::new(kind, self.span.to(joint.span))) diff --git a/compiler/rustc_ast_pretty/src/pprust.rs b/compiler/rustc_ast_pretty/src/pprust.rs index cb48deb5886..aa790878c52 100644 --- a/compiler/rustc_ast_pretty/src/pprust.rs +++ b/compiler/rustc_ast_pretty/src/pprust.rs @@ -289,10 +289,6 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option<Span>) doc_comment_to_string(comment_kind, attr_style, data) } token::Eof => "<eof>".to_string(), - token::Whitespace => " ".to_string(), - token::Comment => "/* */".to_string(), - token::Shebang(s) => format!("/* shebang: {}*/", s), - token::Unknown(s) => s.to_string(), token::Interpolated(ref nt) => nonterminal_to_string(nt), } diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 409784812f5..39c82f97e0a 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -189,7 +189,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)> } OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), + Eof => unreachable!(), } } } diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs index ac395f6cbc2..8dc0db01ecb 100644 --- a/compiler/rustc_parse/src/lexer/unicode_chars.rs +++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs @@ -303,7 +303,7 @@ const UNICODE_ARRAY: &[(char, &str, char)] = &[ // However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add // fancier error recovery to it, as there will be less overall work to do this way. const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[ - (' ', "Space", Some(token::Whitespace)), + (' ', "Space", None), ('_', "Underscore", Some(token::Ident(kw::Underscore, false))), ('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))), (',', "Comma", Some(token::Comma)), diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index bc857c97742..462279b0a9e 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -348,9 +348,6 @@ pub fn tokenstream_probably_equal_for_proc_macro( | token::CloseDelim(DelimToken::NoDelim) // The pretty printer collapses many semicolons into one. | token::Semi - // The pretty printer collapses whitespace arbitrarily and can - // introduce whitespace from `NoDelim`. - | token::Whitespace // The pretty printer can turn `$crate` into `::crate_name` | token::ModSep = token.kind { return false; @@ -506,8 +503,6 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool { | (&Pound, &Pound) | (&Dollar, &Dollar) | (&Question, &Question) - | (&Whitespace, &Whitespace) - | (&Comment, &Comment) | (&Eof, &Eof) => true, (&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b, @@ -516,8 +511,6 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool { (&DocComment(a1, a2, a3), &DocComment(b1, b2, b3)) => a1 == b1 && a2 == b2 && a3 == b3, - (&Shebang(a), &Shebang(b)) => a == b, - (&Literal(a), &Literal(b)) => a == b, (&Lifetime(a), &Lifetime(b)) => a == b, |