Reserve prefixed identifiers and string literals (RFC 3101)

This commit denies any identifiers immediately followed by
one of three tokens `"`, `'` or `#`, which is stricter than
the requirements of RFC 3101 but may be necessary according
to the discussion at [Zulip].

[Zulip]: https://rust-lang.zulipchat.com/#narrow/stream/268952-edition-2021/topic/reserved.20prefixes/near/238470099

1 files changed, 12 insertions, 5 deletions

diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index b9781581ff7..5f3e245bf99 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -66,6 +66,8 @@ pub enum TokenKind {
     Ident,
     /// "r#ident"
     RawIdent,
+    /// `foo#`, `foo'`, `foo"`. Note the tailer is not included.
+    BadPrefix,
     /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
     Literal { kind: LiteralKind, suffix_start: usize },
     /// "'a"
@@ -323,7 +325,7 @@ impl Cursor<'_> {
                     let kind = RawStr { n_hashes, err };
                     Literal { kind, suffix_start }
                 }
-                _ => self.ident(),
+                _ => self.ident_or_bad_prefix(),
             },
 
             // Byte literal, byte string literal, raw byte string literal or identifier.
@@ -358,12 +360,12 @@ impl Cursor<'_> {
                     let kind = RawByteStr { n_hashes, err };
                     Literal { kind, suffix_start }
                 }
-                _ => self.ident(),
+                _ => self.ident_or_bad_prefix(),
             },
 
             // Identifier (this should be checked after other variant that can
             // start as identifier).
-            c if is_id_start(c) => self.ident(),
+            c if is_id_start(c) => self.ident_or_bad_prefix(),
 
             // Numeric literal.
             c @ '0'..='9' => {
@@ -487,11 +489,16 @@ impl Cursor<'_> {
         RawIdent
     }
 
-    fn ident(&mut self) -> TokenKind {
+    fn ident_or_bad_prefix(&mut self) -> TokenKind {
         debug_assert!(is_id_start(self.prev()));
         // Start is already eaten, eat the rest of identifier.
         self.eat_while(is_id_continue);
-        Ident
+        // Good prefixes must have been handled eariler. So if
+        // we see a prefix here, it is definitely a bad prefix.
+        match self.first() {
+            '#' | '"' | '\'' => BadPrefix,
+            _ => Ident,
+        }
     }
 
     fn number(&mut self, first_digit: char) -> LiteralKind {