Small doc improvements.

The phrasing is from the commit description of 395ee0b79f23b90593b01dd0a78451b8c93b0aa6 by @Matklad.

1 files changed, 11 insertions, 2 deletions

diff --git a/src/librustc_lexer/src/lib.rs b/src/librustc_lexer/src/lib.rs
index 5ccfc1b276b..e44feee9660 100644
--- a/src/librustc_lexer/src/lib.rs
+++ b/src/librustc_lexer/src/lib.rs
@@ -1,5 +1,11 @@
 //! Low-level Rust lexer.
 //!
+//! The idea with `librustc_lexer` is to make a reusable library,
+//! by separating out pure lexing and rustc-specific concerns, like spans,
+//! error reporting an interning.  So, rustc_lexer operates directly on `&str`,
+//! produces simple tokens which are a pair of type-tag and a bit of original text,
+//! and does not report errors, instead storing them as flags on the token.
+//!
 //! Tokens produced by this lexer are not yet ready for parsing the Rust syntax,
 //! for that see `librustc_parse::lexer`, which converts this basic token stream
 //! into wide tokens used by actual parser.
@@ -719,6 +725,9 @@ impl Cursor<'_> {
 
             // Check that amount of closing '#' symbols
             // is equal to the amount of opening ones.
+            // Note that this will not consume extra trailing `#` characters:
+            // `r###"abcde"####` is lexed as a `LexedRawString { n_hashes: 3 }`
+            // followed by a `#` token.
             let mut hashes_left = n_start_hashes;
             let is_closing_hash = |c| {
                 if c == '#' && hashes_left != 0 {
@@ -739,8 +748,8 @@ impl Cursor<'_> {
                     possible_terminator_offset: None,
                 };
             } else if n_end_hashes > max_hashes {
-                // Keep track of possible terminators to give a hint about where there might be
-                // a missing terminator
+                // Keep track of possible terminators to give a hint about
+                // where there might be a missing terminator
                 possible_terminator_offset =
                     Some(self.len_consumed() - start_pos - n_end_hashes + prefix_len);
                 max_hashes = n_end_hashes;