Rollup merge of #106566 - clubby789:contiguous-weird-unicode, r=cjgillot

Emit a single error for contiguous sequences of unknown tokens

Closes #106101

On encountering a sequence of identical source characters which are unknown tokens, note the amount of subsequent characters and advance past them silently. The old behavior was to emit an error and 'help' note for every single one.

`@rustbot` label +A-diagnostics +A-parser

8 files changed, 49 insertions, 9 deletions

diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index f027843e6b4..8761c23625b 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -79,7 +79,7 @@ impl<'a> StringReader<'a> {
     /// preceded by whitespace.
     fn next_token(&mut self) -> (Token, bool) {
         let mut preceded_by_whitespace = false;
-
+        let mut swallow_next_invalid = 0;
         // Skip trivial (whitespace & comments) tokens
         loop {
             let token = self.cursor.advance_token();
@@ -232,19 +232,34 @@ impl<'a> StringReader<'a> {
                 rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
 
                 rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => {
-                    let c = self.str_from(start).chars().next().unwrap();
+                    // Don't emit diagnostics for sequences of the same invalid token
+                    if swallow_next_invalid > 0 {
+                        swallow_next_invalid -= 1;
+                        continue;
+                    }
+                    let mut it = self.str_from_to_end(start).chars();
+                    let c = it.next().unwrap();
+                    let repeats = it.take_while(|c1| *c1 == c).count();
                     let mut err =
-                        self.struct_err_span_char(start, self.pos, "unknown start of token", c);
+                        self.struct_err_span_char(start, self.pos + Pos::from_usize(repeats * c.len_utf8()), "unknown start of token", c);
                     // FIXME: the lexer could be used to turn the ASCII version of unicode
                     // homoglyphs, instead of keeping a table in `check_for_substitution`into the
                     // token. Ideally, this should be inside `rustc_lexer`. However, we should
                     // first remove compound tokens like `<<` from `rustc_lexer`, and then add
                     // fancier error recovery to it, as there will be less overall work to do this
                     // way.
-                    let token = unicode_chars::check_for_substitution(self, start, c, &mut err);
+                    let token = unicode_chars::check_for_substitution(self, start, c, &mut err, repeats+1);
                     if c == '\x00' {
                         err.help("source files must contain UTF-8 encoded text, unexpected null bytes might occur when a different encoding is used");
                     }
+                    if repeats > 0 {
+                        if repeats == 1 {
+                            err.note(format!("character appears once more"));
+                        } else {
+                            err.note(format!("character appears {repeats} more times"));
+                        }
+                        swallow_next_invalid = repeats;
+                    }
                     err.emit();
                     if let Some(token) = token {
                         token
@@ -486,6 +501,11 @@ impl<'a> StringReader<'a> {
         &self.src[self.src_index(start)..self.src_index(end)]
     }
 
+    /// Slice of the source text spanning from `start` until the end
+    fn str_from_to_end(&self, start: BytePos) -> &str {
+        &self.src[self.src_index(start)..]
+    }
+
     fn report_raw_str_error(&self, start: BytePos, prefix_len: u32) -> ! {
         match rustc_lexer::validate_raw_str(self.str_from(start), prefix_len) {
             Err(RawStrError::InvalidStarter { bad_char }) => {
diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs
index f1b50296e25..65479b341d7 100644
--- a/compiler/rustc_parse/src/lexer/unicode_chars.rs
+++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs
@@ -337,10 +337,11 @@ pub(super) fn check_for_substitution<'a>(
     pos: BytePos,
     ch: char,
     err: &mut Diagnostic,
+    count: usize,
 ) -> Option<token::TokenKind> {
     let &(_u_char, u_name, ascii_char) = UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch)?;
 
-    let span = Span::with_root_ctxt(pos, pos + Pos::from_usize(ch.len_utf8()));
+    let span = Span::with_root_ctxt(pos, pos + Pos::from_usize(ch.len_utf8() * count));
 
     let Some((_ascii_char, ascii_name, token)) = ASCII_ARRAY.iter().find(|&&(c, _, _)| c == ascii_char) else {
         let msg = format!("substitution character not found for '{}'", ch);
@@ -369,7 +370,12 @@ pub(super) fn check_for_substitution<'a>(
             "Unicode character '{}' ({}) looks like '{}' ({}), but it is not",
             ch, u_name, ascii_char, ascii_name
         );
-        err.span_suggestion(span, &msg, ascii_char, Applicability::MaybeIncorrect);
+        err.span_suggestion(
+            span,
+            &msg,
+            ascii_char.to_string().repeat(count),
+            Applicability::MaybeIncorrect,
+        );
     }
     token.clone()
 }
diff --git a/tests/rustdoc-ui/invalid-syntax.stderr b/tests/rustdoc-ui/invalid-syntax.stderr
index 597d19e748c..6140a06c555 100644
--- a/tests/rustdoc-ui/invalid-syntax.stderr
+++ b/tests/rustdoc-ui/invalid-syntax.stderr
@@ -77,8 +77,6 @@ LL | ///     ```
    |         ^^^
    |
    = note: error from rustc: unknown start of token: `
-   = note: error from rustc: unknown start of token: `
-   = note: error from rustc: unknown start of token: `
 
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:64:5
diff --git a/tests/ui/parser/issues/issue-66473.stderr b/tests/ui/parser/issues/issue-66473.stderr
index 8a16d7f9551..0e8b0a5da22 100644
--- a/tests/ui/parser/issues/issue-66473.stderr
+++ b/tests/ui/parser/issues/issue-66473.stderr
Binary files differdiff --git a/tests/ui/parser/issues/issue-68629.stderr b/tests/ui/parser/issues/issue-68629.stderr
index b2c7dddc801..43a903e6c46 100644
--- a/tests/ui/parser/issues/issue-68629.stderr
+++ b/tests/ui/parser/issues/issue-68629.stderr
Binary files differdiff --git a/tests/ui/parser/issues/issue-68730.stderr b/tests/ui/parser/issues/issue-68730.stderr
index 6585a19d954..5bca5bbebea 100644
--- a/tests/ui/parser/issues/issue-68730.stderr
+++ b/tests/ui/parser/issues/issue-68730.stderr
Binary files differdiff --git a/tests/ui/parser/unicode-chars.rs b/tests/ui/parser/unicode-chars.rs
index 89ae85ec990..ba35e95c82a 100644
--- a/tests/ui/parser/unicode-chars.rs
+++ b/tests/ui/parser/unicode-chars.rs
@@ -2,4 +2,8 @@ fn main() {
     let y = 0;
     //~^ ERROR unknown start of token: \u{37e}
     //~^^ HELP Unicode character ';' (Greek Question Mark) looks like ';' (Semicolon), but it is not
+        let x = 0;
+    //~^ ERROR unknown start of token: \u{a0}
+    //~^^ NOTE character appears 3 more times
+    //~^^^ HELP Unicode character ' ' (No-Break Space) looks like ' ' (Space), but it is not
 }
diff --git a/tests/ui/parser/unicode-chars.stderr b/tests/ui/parser/unicode-chars.stderr
index 0cfe9240e85..6a5b27872e7 100644
--- a/tests/ui/parser/unicode-chars.stderr
+++ b/tests/ui/parser/unicode-chars.stderr
@@ -9,5 +9,17 @@ help: Unicode character ';' (Greek Question Mark) looks like ';' (Semicolon), b
 LL |     let y = 0;
    |              ~
 
-error: aborting due to previous error
+error: unknown start of token: \u{a0}
+  --> $DIR/unicode-chars.rs:5:5
+   |
+LL |         let x = 0;
+   |     ^^^^
+   |
+   = note: character appears 3 more times
+help: Unicode character ' ' (No-Break Space) looks like ' ' (Space), but it is not
+   |
+LL |         let x = 0;
+   |     ++++
+
+error: aborting due to 2 previous errors