4 files changed, 44 insertions, 19 deletions
diff --git a/compiler/rustc_lexer/src/cursor.rs b/compiler/rustc_lexer/src/cursor.rs
index d173c3ac032..e0e3bd0e30b 100644
--- a/compiler/rustc_lexer/src/cursor.rs
+++ b/compiler/rustc_lexer/src/cursor.rs
@@ -103,4 +103,11 @@ impl<'a> Cursor<'a> {
             self.bump();
         }
     }
+
+    pub(crate) fn eat_until(&mut self, byte: u8) {
+        self.chars = match memchr::memchr(byte, self.as_str().as_bytes()) {
+            Some(index) => self.as_str()[index..].chars(),
+            None => "".chars(),
+        }
+    }
 }
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index aa4abf678b9..bf18845a083 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -291,7 +291,7 @@ pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError>
 }
 
 /// Creates an iterator that produces tokens from the input string.
-pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ {
+pub fn tokenize(input: &str) -> impl Iterator<Item = Token> {
     let mut cursor = Cursor::new(input);
     std::iter::from_fn(move || {
         let token = cursor.advance_token();
@@ -483,7 +483,7 @@ impl Cursor<'_> {
             _ => None,
         };
 
-        self.eat_while(|c| c != '\n');
+        self.eat_until(b'\n');
         LineComment { doc_style }
     }
 
@@ -888,7 +888,7 @@ impl Cursor<'_> {
         // Skip the string contents and on each '#' character met, check if this is
         // a raw string termination.
         loop {
-            self.eat_while(|c| c != '"');
+            self.eat_until(b'"');
 
             if self.is_eof() {
                 return Err(RawStrError::NoTerminator {
diff --git a/compiler/rustc_lexer/src/tests.rs b/compiler/rustc_lexer/src/tests.rs
index db7225fc2a8..8203ae70b07 100644
--- a/compiler/rustc_lexer/src/tests.rs
+++ b/compiler/rustc_lexer/src/tests.rs
@@ -131,7 +131,9 @@ fn check_lexing(src: &str, expect: Expect) {
 
 #[test]
 fn smoke_test() {
-    check_lexing("/* my source file */ fn main() { println!(\"zebra\"); }\n", expect![[r#"
+    check_lexing(
+        "/* my source file */ fn main() { println!(\"zebra\"); }\n",
+        expect![[r#"
             Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 }
             Token { kind: Whitespace, len: 1 }
             Token { kind: Ident, len: 2 }
@@ -151,7 +153,8 @@ fn smoke_test() {
             Token { kind: Whitespace, len: 1 }
             Token { kind: CloseBrace, len: 1 }
             Token { kind: Whitespace, len: 1 }
-        "#]])
+        "#]],
+    )
 }
 
 #[test]
@@ -194,35 +197,47 @@ fn comment_flavors() {
 
 #[test]
 fn nested_block_comments() {
-    check_lexing("/* /* */ */'a'", expect![[r#"
+    check_lexing(
+        "/* /* */ */'a'",
+        expect![[r#"
             Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
             Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
-        "#]])
+        "#]],
+    )
 }
 
 #[test]
 fn characters() {
-    check_lexing("'a' ' ' '\\n'", expect![[r#"
+    check_lexing(
+        "'a' ' ' '\\n'",
+        expect![[r#"
             Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
             Token { kind: Whitespace, len: 1 }
             Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
             Token { kind: Whitespace, len: 1 }
             Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 }
-        "#]]);
+        "#]],
+    );
 }
 
 #[test]
 fn lifetime() {
-    check_lexing("'abc", expect![[r#"
+    check_lexing(
+        "'abc",
+        expect![[r#"
             Token { kind: Lifetime { starts_with_number: false }, len: 4 }
-        "#]]);
+        "#]],
+    );
 }
 
 #[test]
 fn raw_string() {
-    check_lexing("r###\"\"#a\\b\x00c\"\"###", expect![[r#"
+    check_lexing(
+        "r###\"\"#a\\b\x00c\"\"###",
+        expect![[r#"
             Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 17 }, len: 17 }
-        "#]])
+        "#]],
+    )
 }
 
 #[test]
diff --git a/compiler/rustc_lexer/src/unescape/tests.rs b/compiler/rustc_lexer/src/unescape/tests.rs
index 6fa7a150516..5b99495f475 100644
--- a/compiler/rustc_lexer/src/unescape/tests.rs
+++ b/compiler/rustc_lexer/src/unescape/tests.rs
@@ -108,12 +108,15 @@ fn test_unescape_str_warn() {
     check("\\\n", &[]);
     check("\\\n ", &[]);
 
-    check("\\\n \u{a0} x", &[
-        (0..5, Err(EscapeError::UnskippedWhitespaceWarning)),
-        (3..5, Ok('\u{a0}')),
-        (5..6, Ok(' ')),
-        (6..7, Ok('x')),
-    ]);
+    check(
+        "\\\n \u{a0} x",
+        &[
+            (0..5, Err(EscapeError::UnskippedWhitespaceWarning)),
+            (3..5, Ok('\u{a0}')),
+            (5..6, Ok(' ')),
+            (6..7, Ok('x')),
+        ],
+    );
     check("\\\n  \n  x", &[(0..7, Err(EscapeError::MultipleSkippedLinesWarning)), (7..8, Ok('x'))]);
 }