From 45a1e492b1adfc6f7664d1da736dff147e5c3168 Mon Sep 17 00:00:00 2001
From: Ed Page <eopage@gmail.com>
Date: Wed, 9 Jul 2025 16:40:22 -0500
Subject: feat(lexer): Allow including frontmatter with 'tokenize'

---
 src/tools/rust-analyzer/crates/parser/src/lexed_str.rs         | 10 ++++++----
 .../rust-analyzer/crates/proc-macro-srv/src/server_impl.rs     |  2 +-
 src/tools/rust-analyzer/crates/tt/src/lib.rs                   |  2 +-
 3 files changed, 8 insertions(+), 6 deletions(-)

(limited to 'src/tools/rust-analyzer/crates')

diff --git a/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs b/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs
index e6c92dec681..bff9acd78fa 100644
--- a/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs
@@ -11,8 +11,8 @@
 use std::ops;
 
 use rustc_literal_escaper::{
-    EscapeError, Mode, unescape_byte, unescape_byte_str, unescape_c_str, unescape_char,
-    unescape_str,
+    unescape_byte, unescape_byte_str, unescape_c_str, unescape_char, unescape_str, EscapeError,
+    Mode,
 };
 
 use crate::{
@@ -44,7 +44,9 @@ impl<'a> LexedStr<'a> {
 
         // Re-create the tokenizer from scratch every token because `GuardedStrPrefix` is one token in the lexer
         // but we want to split it to two in edition <2024.
-        while let Some(token) = rustc_lexer::tokenize(&text[conv.offset..]).next() {
+        while let Some(token) =
+            rustc_lexer::tokenize(&text[conv.offset..], rustc_lexer::FrontmatterAllowed::No).next()
+        {
             let token_text = &text[conv.offset..][..token.len as usize];
 
             conv.extend_token(&token.kind, token_text);
@@ -58,7 +60,7 @@ impl<'a> LexedStr<'a> {
             return None;
         }
 
-        let token = rustc_lexer::tokenize(text).next()?;
+        let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next()?;
         if token.len as usize != text.len() {
             return None;
         }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs
index dd576f23ae9..662f6257642 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs
@@ -121,7 +121,7 @@ pub(super) fn literal_from_str<Span: Copy>(
     use proc_macro::bridge::LitKind;
     use rustc_lexer::{LiteralKind, Token, TokenKind};
 
-    let mut tokens = rustc_lexer::tokenize(s);
+    let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No);
     let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 });
 
     let lit = if minus_or_lit.kind == TokenKind::Minus {
diff --git a/src/tools/rust-analyzer/crates/tt/src/lib.rs b/src/tools/rust-analyzer/crates/tt/src/lib.rs
index 14574a6456b..44123385c8c 100644
--- a/src/tools/rust-analyzer/crates/tt/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/tt/src/lib.rs
@@ -579,7 +579,7 @@ where
 {
     use rustc_lexer::LiteralKind;
 
-    let token = rustc_lexer::tokenize(text).next_tuple();
+    let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple();
     let Some((rustc_lexer::Token {
         kind: rustc_lexer::TokenKind::Literal { kind, suffix_start },
         ..
-- 
cgit 1.4.1-3-g733a5