Auto merge of #136585 - gvozdvmozgu:memchr-eat-until-lexer, r=lcnr

implement `eat_until` leveraging memchr in lexer

4 files changed, 11 insertions, 2 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 7c8f06a0239..89eb514917d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4064,6 +4064,7 @@ name = "rustc_lexer"
 version = "0.0.0"
 dependencies = [
  "expect-test",
+ "memchr",
  "unicode-properties",
  "unicode-xid",
 ]
diff --git a/compiler/rustc_lexer/Cargo.toml b/compiler/rustc_lexer/Cargo.toml
index 84b9e292295..4b3492fdeda 100644
--- a/compiler/rustc_lexer/Cargo.toml
+++ b/compiler/rustc_lexer/Cargo.toml
@@ -14,6 +14,7 @@ Rust lexer used by rustc. No stability guarantees are provided.
 
 # Note that this crate purposefully does not depend on other rustc crates
 [dependencies]
+memchr = "2.7.4"
 unicode-xid = "0.2.0"
 
 [dependencies.unicode-properties]
diff --git a/compiler/rustc_lexer/src/cursor.rs b/compiler/rustc_lexer/src/cursor.rs
index d173c3ac032..e0e3bd0e30b 100644
--- a/compiler/rustc_lexer/src/cursor.rs
+++ b/compiler/rustc_lexer/src/cursor.rs
@@ -103,4 +103,11 @@ impl<'a> Cursor<'a> {
             self.bump();
         }
     }
+
+    pub(crate) fn eat_until(&mut self, byte: u8) {
+        self.chars = match memchr::memchr(byte, self.as_str().as_bytes()) {
+            Some(index) => self.as_str()[index..].chars(),
+            None => "".chars(),
+        }
+    }
 }
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index aa4abf678b9..c63ab77deca 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -483,7 +483,7 @@ impl Cursor<'_> {
             _ => None,
         };
 
-        self.eat_while(|c| c != '\n');
+        self.eat_until(b'\n');
         LineComment { doc_style }
     }
 
@@ -888,7 +888,7 @@ impl Cursor<'_> {
         // Skip the string contents and on each '#' character met, check if this is
         // a raw string termination.
         loop {
-            self.eat_while(|c| c != '"');
+            self.eat_until(b'"');
 
             if self.is_eof() {
                 return Err(RawStrError::NoTerminator {