Auto merge of #14837 - Veykril:rustc-lexer, r=Veykril

Support c string literals

6 files changed, 100 insertions, 20 deletions

diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs
index c43d0830b9e..1eef2861165 100644
--- a/crates/syntax/src/ast/expr_ext.rs
+++ b/crates/syntax/src/ast/expr_ext.rs
@@ -288,6 +288,7 @@ impl ast::ArrayExpr {
 pub enum LiteralKind {
     String(ast::String),
     ByteString(ast::ByteString),
+    CString(ast::CString),
     IntNumber(ast::IntNumber),
     FloatNumber(ast::FloatNumber),
     Char(ast::Char),
@@ -319,6 +320,9 @@ impl ast::Literal {
         if let Some(t) = ast::ByteString::cast(token.clone()) {
             return LiteralKind::ByteString(t);
         }
+        if let Some(t) = ast::CString::cast(token.clone()) {
+            return LiteralKind::CString(t);
+        }
         if let Some(t) = ast::Char::cast(token.clone()) {
             return LiteralKind::Char(t);
         }
diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs
index a3209c5abd2..f5863e9efe0 100644
--- a/crates/syntax/src/ast/generated/tokens.rs
+++ b/crates/syntax/src/ast/generated/tokens.rs
@@ -91,6 +91,27 @@ impl AstToken for ByteString {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CString {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for CString {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for CString {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == C_STRING }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct IntNumber {
     pub(crate) syntax: SyntaxToken,
 }
diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs
index 2cd312e7f4f..090eb89f470 100644
--- a/crates/syntax/src/ast/token_ext.rs
+++ b/crates/syntax/src/ast/token_ext.rs
@@ -145,6 +145,10 @@ impl QuoteOffsets {
 }
 
 pub trait IsString: AstToken {
+    const RAW_PREFIX: &'static str;
+    fn is_raw(&self) -> bool {
+        self.text().starts_with(Self::RAW_PREFIX)
+    }
     fn quote_offsets(&self) -> Option<QuoteOffsets> {
         let text = self.text();
         let offsets = QuoteOffsets::new(text)?;
@@ -183,20 +187,18 @@ pub trait IsString: AstToken {
             cb(text_range + offset, unescaped_char);
         });
     }
-}
-
-impl IsString for ast::String {}
-
-impl ast::String {
-    pub fn is_raw(&self) -> bool {
-        self.text().starts_with('r')
-    }
-    pub fn map_range_up(&self, range: TextRange) -> Option<TextRange> {
+    fn map_range_up(&self, range: TextRange) -> Option<TextRange> {
         let contents_range = self.text_range_between_quotes()?;
         assert!(TextRange::up_to(contents_range.len()).contains_range(range));
         Some(range + contents_range.start())
     }
+}
 
+impl IsString for ast::String {
+    const RAW_PREFIX: &'static str = "r";
+}
+
+impl ast::String {
     pub fn value(&self) -> Option<Cow<'_, str>> {
         if self.is_raw() {
             let text = self.text();
@@ -235,13 +237,11 @@ impl ast::String {
     }
 }
 
-impl IsString for ast::ByteString {}
+impl IsString for ast::ByteString {
+    const RAW_PREFIX: &'static str = "br";
+}
 
 impl ast::ByteString {
-    pub fn is_raw(&self) -> bool {
-        self.text().starts_with("br")
-    }
-
     pub fn value(&self) -> Option<Cow<'_, [u8]>> {
         if self.is_raw() {
             let text = self.text();
@@ -280,6 +280,49 @@ impl ast::ByteString {
     }
 }
 
+impl IsString for ast::CString {
+    const RAW_PREFIX: &'static str = "cr";
+}
+
+impl ast::CString {
+    pub fn value(&self) -> Option<Cow<'_, str>> {
+        if self.is_raw() {
+            let text = self.text();
+            let text =
+                &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
+            return Some(Cow::Borrowed(text));
+        }
+
+        let text = self.text();
+        let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
+
+        let mut buf = String::new();
+        let mut prev_end = 0;
+        let mut has_error = false;
+        unescape_literal(text, Mode::Str, &mut |char_range, unescaped_char| match (
+            unescaped_char,
+            buf.capacity() == 0,
+        ) {
+            (Ok(c), false) => buf.push(c),
+            (Ok(_), true) if char_range.len() == 1 && char_range.start == prev_end => {
+                prev_end = char_range.end
+            }
+            (Ok(c), true) => {
+                buf.reserve_exact(text.len());
+                buf.push_str(&text[..prev_end]);
+                buf.push(c);
+            }
+            (Err(_), _) => has_error = true,
+        });
+
+        match (has_error, buf.capacity() == 0) {
+            (true, _) => None,
+            (false, true) => Some(Cow::Borrowed(text)),
+            (false, false) => Some(Cow::Owned(buf)),
+        }
+    }
+}
+
 impl ast::IntNumber {
     pub fn radix(&self) -> Radix {
         match self.text().get(..2).unwrap_or_default() {
diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs
index f3d644c680e..45e59160982 100644
--- a/crates/syntax/src/parsing/reparsing.rs
+++ b/crates/syntax/src/parsing/reparsing.rs
@@ -39,7 +39,7 @@ fn reparse_token(
     let prev_token = root.covering_element(edit.delete).as_token()?.clone();
     let prev_token_kind = prev_token.kind();
     match prev_token_kind {
-        WHITESPACE | COMMENT | IDENT | STRING => {
+        WHITESPACE | COMMENT | IDENT | STRING | BYTE_STRING | C_STRING => {
             if prev_token_kind == WHITESPACE || prev_token_kind == COMMENT {
                 // removing a new line may extends previous token
                 let deleted_range = edit.delete - prev_token.text_range().start();
diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs
index 77a8363a185..c49c5fa108b 100644
--- a/crates/syntax/src/tests/sourcegen_ast.rs
+++ b/crates/syntax/src/tests/sourcegen_ast.rs
@@ -573,10 +573,11 @@ impl Field {
 
 fn lower(grammar: &Grammar) -> AstSrc {
     let mut res = AstSrc {
-        tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident"
-            .split_ascii_whitespace()
-            .map(|it| it.to_string())
-            .collect::<Vec<_>>(),
+        tokens:
+            "Whitespace Comment String ByteString CString IntNumber FloatNumber Char Byte Ident"
+                .split_ascii_whitespace()
+                .map(|it| it.to_string())
+                .collect::<Vec<_>>(),
         ..Default::default()
     };
 
diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs
index 089ad74dadc..e0ec6a242ff 100644
--- a/crates/syntax/src/validation.rs
+++ b/crates/syntax/src/validation.rs
@@ -9,7 +9,7 @@ use rustc_lexer::unescape::{self, unescape_literal, Mode};
 
 use crate::{
     algo,
-    ast::{self, HasAttrs, HasVisibility},
+    ast::{self, HasAttrs, HasVisibility, IsString},
     match_ast, AstNode, SyntaxError,
     SyntaxKind::{CONST, FN, INT_NUMBER, TYPE_ALIAS},
     SyntaxNode, SyntaxToken, TextSize, T,
@@ -156,6 +156,17 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
                 }
             }
         }
+        ast::LiteralKind::CString(s) => {
+            if !s.is_raw() {
+                if let Some(without_quotes) = unquote(text, 2, '"') {
+                    unescape_literal(without_quotes, Mode::ByteStr, &mut |range, char| {
+                        if let Err(err) = char {
+                            push_err(1, range.start, err);
+                        }
+                    });
+                }
+            }
+        }
         ast::LiteralKind::Char(_) => {
             if let Some(without_quotes) = unquote(text, 1, '\'') {
                 unescape_literal(without_quotes, Mode::Char, &mut |range, char| {