From 5a68abb0943e687cc6c8165376b4ed6deda17db3 Mon Sep 17 00:00:00 2001 From: Esteban Kuber Date: Sun, 29 Aug 2021 08:34:23 +0000 Subject: Tokenize emoji as if they were valid indentifiers In the lexer, consider emojis to be valid identifiers and reject them later to avoid knock down parse errors. --- compiler/rustc_parse/src/lexer/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index cf35c3cd53b..c4a3dd9bfda 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -222,6 +222,12 @@ impl<'a> StringReader<'a> { } token::Ident(sym, is_raw_ident) } + rustc_lexer::TokenKind::InvalidIdent => { + let sym = nfc_normalize(self.str_from(start)); + let span = self.mk_sp(start, self.pos); + self.sess.bad_unicode_identifiers.borrow_mut().entry(sym).or_default().push(span); + token::Ident(sym, false) + } rustc_lexer::TokenKind::Literal { kind, suffix_start } => { let suffix_start = start + BytePos(suffix_start as u32); let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind); -- cgit 1.4.1-3-g733a5