Improve error messages for raw strings (#60762)

This diff improves error messages around raw strings in a few ways:
- Catch extra trailing `#` in the parser. This can't be handled in the lexer because we could be in a macro that actually expects another # (see test)
- Refactor & unify error handling in the lexer between ByteStrings and RawByteStrings
- Detect potentially intended terminators (longest sequence of "#*" is suggested)

2 files changed, 90 insertions, 35 deletions

diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs
index ac58cbb9e8d..2f720d95c6d 100644
--- a/src/librustc_parse/lexer/mod.rs
+++ b/src/librustc_parse/lexer/mod.rs
@@ -1,20 +1,20 @@
 use rustc_ast::token::{self, Token, TokenKind};
 use rustc_ast::util::comments;
 use rustc_data_structures::sync::Lrc;
-use rustc_errors::{error_code, DiagnosticBuilder, FatalError};
-use rustc_lexer::unescape;
+use rustc_errors::{error_code, Applicability, DiagnosticBuilder, FatalError};
 use rustc_lexer::Base;
+use rustc_lexer::{unescape, LexRawStrError, UnvalidatedRawStr, ValidatedRawStr};
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{BytePos, Pos, Span};
 
 use log::debug;
 use std::char;
-use std::convert::TryInto;
 
 mod tokentrees;
 mod unescape_error_reporting;
 mod unicode_chars;
+
 use unescape_error_reporting::{emit_unescape_error, push_escaped_char};
 
 #[derive(Clone, Debug)]
@@ -376,30 +376,22 @@ impl<'a> StringReader<'a> {
                 let id = self.symbol_from_to(content_start, content_end);
                 (token::ByteStr, id)
             }
-            rustc_lexer::LiteralKind::RawStr { n_hashes, started, terminated } => {
-                if !started {
-                    self.report_non_started_raw_string(start);
-                }
-                if !terminated {
-                    self.report_unterminated_raw_string(start, n_hashes)
-                }
-                let n_hashes: u16 = self.restrict_n_hashes(start, n_hashes);
+            rustc_lexer::LiteralKind::RawStr(unvalidated_raw_str) => {
+                let valid_raw_str = self.validate_and_report_errors(start, unvalidated_raw_str);
+                let n_hashes = valid_raw_str.num_hashes();
                 let n = u32::from(n_hashes);
+
                 let content_start = start + BytePos(2 + n);
                 let content_end = suffix_start - BytePos(1 + n);
                 self.validate_raw_str_escape(content_start, content_end);
                 let id = self.symbol_from_to(content_start, content_end);
                 (token::StrRaw(n_hashes), id)
             }
-            rustc_lexer::LiteralKind::RawByteStr { n_hashes, started, terminated } => {
-                if !started {
-                    self.report_non_started_raw_string(start);
-                }
-                if !terminated {
-                    self.report_unterminated_raw_string(start, n_hashes)
-                }
-                let n_hashes: u16 = self.restrict_n_hashes(start, n_hashes);
+            rustc_lexer::LiteralKind::RawByteStr(unvalidated_raw_str) => {
+                let validated_raw_str = self.validate_and_report_errors(start, unvalidated_raw_str);
+                let n_hashes = validated_raw_str.num_hashes();
                 let n = u32::from(n_hashes);
+
                 let content_start = start + BytePos(3 + n);
                 let content_end = suffix_start - BytePos(1 + n);
                 self.validate_raw_byte_str_escape(content_start, content_end);
@@ -485,6 +477,26 @@ impl<'a> StringReader<'a> {
         }
     }
 
+    fn validate_and_report_errors(
+        &self,
+        start: BytePos,
+        unvalidated_raw_str: UnvalidatedRawStr,
+    ) -> ValidatedRawStr {
+        match unvalidated_raw_str.validate() {
+            Err(LexRawStrError::InvalidStarter) => self.report_non_started_raw_string(start),
+            Err(LexRawStrError::NoTerminator { expected, found, possible_terminator_offset }) => {
+                self.report_unterminated_raw_string(
+                    start,
+                    expected,
+                    possible_terminator_offset,
+                    found,
+                )
+            }
+            Err(LexRawStrError::TooManyDelimiters) => self.report_too_many_hashes(start),
+            Ok(valid) => valid,
+        }
+    }
+
     fn report_non_started_raw_string(&self, start: BytePos) -> ! {
         let bad_char = self.str_from(start).chars().last().unwrap();
         self.struct_fatal_span_char(
@@ -498,38 +510,52 @@ impl<'a> StringReader<'a> {
         FatalError.raise()
     }
 
-    fn report_unterminated_raw_string(&self, start: BytePos, n_hashes: usize) -> ! {
+    fn report_unterminated_raw_string(
+        &self,
+        start: BytePos,
+        n_hashes: usize,
+        possible_offset: Option<usize>,
+        found_terminators: usize,
+    ) -> ! {
         let mut err = self.sess.span_diagnostic.struct_span_fatal_with_code(
             self.mk_sp(start, start),
             "unterminated raw string",
             error_code!(E0748),
         );
+
         err.span_label(self.mk_sp(start, start), "unterminated raw string");
 
         if n_hashes > 0 {
             err.note(&format!(
                 "this raw string should be terminated with `\"{}`",
-                "#".repeat(n_hashes as usize)
+                "#".repeat(n_hashes)
             ));
         }
 
+        if let Some(possible_offset) = possible_offset {
+            let span = self.mk_sp(
+                start + BytePos(possible_offset as u32),
+                start + BytePos(possible_offset as u32) + BytePos(found_terminators as u32),
+            );
+            err.span_suggestion(
+                span,
+                "you might have intended to terminate the string here",
+                "#".repeat(n_hashes),
+                Applicability::MaybeIncorrect,
+            );
+        }
+
         err.emit();
         FatalError.raise()
     }
 
-    fn restrict_n_hashes(&self, start: BytePos, n_hashes: usize) -> u16 {
-        match n_hashes.try_into() {
-            Ok(n_hashes) => n_hashes,
-            Err(_) => {
-                self.fatal_span_(
-                    start,
-                    self.pos,
-                    "too many `#` symbols: raw strings may be \
-                                  delimited by up to 65535 `#` symbols",
-                )
-                .raise();
-            }
-        }
+    fn report_too_many_hashes(&self, start: BytePos) -> ! {
+        self.fatal_span_(
+            start,
+            self.pos,
+            "too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
+        )
+        .raise();
     }
 
     fn validate_char_escape(&self, content_start: BytePos, content_end: BytePos) {
diff --git a/src/librustc_parse/parser/diagnostics.rs b/src/librustc_parse/parser/diagnostics.rs
index c4546dedfcd..7b6840307cb 100644
--- a/src/librustc_parse/parser/diagnostics.rs
+++ b/src/librustc_parse/parser/diagnostics.rs
@@ -6,7 +6,7 @@ use rustc_ast::ast::{
 };
 use rustc_ast::ast::{AttrVec, ItemKind, Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind};
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, TokenKind};
+use rustc_ast::token::{self, Lit, LitKind, Token, TokenKind};
 use rustc_ast::util::parser::AssocOp;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashSet;
@@ -255,6 +255,10 @@ impl<'a> Parser<'a> {
             }
         }
 
+        if self.check_too_many_raw_str_terminators(&mut err) {
+            return Err(err);
+        }
+
         let sm = self.sess.source_map();
         if self.prev_token.span == DUMMY_SP {
             // Account for macro context where the previous span might not be
@@ -282,6 +286,31 @@ impl<'a> Parser<'a> {
         Err(err)
     }
 
+    fn check_too_many_raw_str_terminators(&mut self, err: &mut DiagnosticBuilder<'_>) -> bool {
+        let prev_token_raw_str = match self.prev_token {
+            Token { kind: TokenKind::Literal(Lit { kind: LitKind::StrRaw(n), .. }), .. } => Some(n),
+            Token {
+                kind: TokenKind::Literal(Lit { kind: LitKind::ByteStrRaw(n), .. }), ..
+            } => Some(n),
+            _ => None,
+        };
+
+        if let Some(n_hashes) = prev_token_raw_str {
+            if self.token.kind == TokenKind::Pound {
+                err.set_primary_message("too many `#` when terminating raw string");
+                err.span_suggestion(
+                    self.token.span,
+                    "Remove the extra `#`",
+                    String::new(),
+                    Applicability::MachineApplicable,
+                );
+                err.note(&format!("The raw string started with {} `#`s", n_hashes));
+                return true;
+            }
+        }
+        false
+    }
+
     pub fn maybe_annotate_with_ascription(
         &mut self,
         err: &mut DiagnosticBuilder<'_>,