diff options
| author | Aleksey Kladov <aleksey.kladov@gmail.com> | 2019-07-30 12:31:41 +0300 |
|---|---|---|
| committer | Aleksey Kladov <aleksey.kladov@gmail.com> | 2019-08-05 13:15:11 +0300 |
| commit | 58ac81a60fe11868b0748a406d8e0b97efa4e8c5 (patch) | |
| tree | df4e02648aad8170cb3758067792b9bdb7f1489a /src/libsyntax/parse | |
| parent | b5e35b128efeed4bfdb4b1ee9d0697389ec9f164 (diff) | |
| download | rust-58ac81a60fe11868b0748a406d8e0b97efa4e8c5.tar.gz rust-58ac81a60fe11868b0748a406d8e0b97efa4e8c5.zip | |
add unknown token
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/tokentrees.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 4 |
3 files changed, 5 insertions, 3 deletions
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index c209ae1cb9f..e86d4c7fde6 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -354,7 +354,7 @@ impl<'a> StringReader<'a> { // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it, // as there will be less overall work to do this way. let token = unicode_chars::check_for_substitution(self, start, c, &mut err) - .unwrap_or(token::Whitespace); + .unwrap_or_else(|| token::Unknown(self.symbol_from(start))); err.emit(); token } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 830fbec58de..37e67a2729e 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -217,7 +217,7 @@ impl<'a> TokenTreesReader<'a> { loop { let token = self.string_reader.next_token(); match token.kind { - token::Whitespace | token::Comment | token::Shebang(_) => { + token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => { self.joint_to_prev = NonJoint; } _ => { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 73adb5c947c..be800b4de66 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -255,6 +255,8 @@ pub enum TokenKind { /// A comment. Comment, Shebang(ast::Name), + /// A completely invalid token which should be skipped. + Unknown(ast::Name), Eof, } @@ -603,7 +605,7 @@ impl Token { DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | - Whitespace | Comment | Shebang(..) | Eof => return None, + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, }; Some(Token::new(kind, self.span.to(joint.span))) |