Rollup merge of #103598 - tshepang:token-kind-docs, r=jackh726

rustc_lexer::TokenKind improve docs

1 files changed, 18 insertions, 5 deletions

diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index c71e6ffe34d..51515976e4e 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -57,29 +57,42 @@ pub enum TokenKind {
     // Multi-char tokens:
     /// "// comment"
     LineComment { doc_style: Option<DocStyle> },
+
     /// `/* block comment */`
     ///
-    /// Block comments can be recursive, so the sequence like `/* /* */`
+    /// Block comments can be recursive, so a sequence like `/* /* */`
     /// will not be considered terminated and will result in a parsing error.
     BlockComment { doc_style: Option<DocStyle>, terminated: bool },
-    /// Any whitespace characters sequence.
+
+    /// Any whitespace character sequence.
     Whitespace,
+
     /// "ident" or "continue"
-    /// At this step keywords are also considered identifiers.
+    ///
+    /// At this step, keywords are also considered identifiers.
     Ident,
+
     /// Like the above, but containing invalid unicode codepoints.
     InvalidIdent,
+
     /// "r#ident"
     RawIdent,
-    /// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the
+
+    /// An unknown prefix, like `foo#`, `foo'`, `foo"`.
+    ///
+    /// Note that only the
     /// prefix (`foo`) is included in the token, not the separator (which is
     /// lexed as its own distinct token). In Rust 2021 and later, reserved
     /// prefixes are reported as errors; in earlier editions, they result in a
     /// (allowed by default) lint, and are treated as regular identifier
     /// tokens.
     UnknownPrefix,
-    /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
+
+    /// Examples: `"12_u8"`, `"1.0e-40"`, `b"123`.
+    ///
+    /// See [LiteralKind] for more details.
     Literal { kind: LiteralKind, suffix_start: u32 },
+
     /// "'a"
     Lifetime { starts_with_number: bool },