diff options
| -rw-r--r-- | crates/mbe/src/subtree_source.rs | 22 | ||||
| -rw-r--r-- | crates/syntax/src/parsing/lexer.rs | 23 |
2 files changed, 23 insertions, 22 deletions
diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs index 226dc3bec82..396ce8b162f 100644 --- a/crates/mbe/src/subtree_source.rs +++ b/crates/mbe/src/subtree_source.rs @@ -2,7 +2,7 @@ use parser::{Token, TokenSource}; use std::cell::{Cell, Ref, RefCell}; -use syntax::{tokenize, SmolStr, SyntaxKind, SyntaxKind::*, T}; +use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; use tt::buffer::{Cursor, TokenBuffer}; #[derive(Debug, Clone, Eq, PartialEq)] @@ -155,17 +155,15 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken { } fn convert_literal(l: &tt::Literal) -> TtToken { - let mut kinds = tokenize(&l.text).0.into_iter().map(|token| token.kind); - - let kind = match kinds.next() { - Some(kind) if kind.is_literal() => Some(kind), - Some(SyntaxKind::MINUS) => match kinds.next() { - Some(kind) if kind.is_literal() => Some(kind), - _ => None, - }, - _ => None, - } - .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); + let is_negated = l.text.starts_with('-'); + let inner_text = &l.text[if is_negated { 1 } else { 0 }..]; + + let kind = lex_single_syntax_kind(inner_text) + .map(|(kind, _error)| kind) + .filter(|kind| { + kind.is_literal() && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER)) + }) + .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); TtToken { kind, is_joint_to_next: false, text: l.text.clone() } } diff --git a/crates/syntax/src/parsing/lexer.rs b/crates/syntax/src/parsing/lexer.rs index f1202113ba7..7e38c32cc5e 100644 --- a/crates/syntax/src/parsing/lexer.rs +++ b/crates/syntax/src/parsing/lexer.rs @@ -1,10 +1,10 @@ //! Lexer analyzes raw input string and produces lexemes (tokens). //! It is just a bridge to `rustc_lexer`. -use rustc_lexer::{LiteralKind as LK, RawStrError}; - use std::convert::TryInto; +use rustc_lexer::{LiteralKind as LK, RawStrError}; + use crate::{ SyntaxError, SyntaxKind::{self, *}, @@ -61,17 +61,18 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) { (tokens, errors) } -/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token -/// encountered at the beginning of the string. +/// Returns `SyntaxKind` and `Option<SyntaxError>` if `text` parses as a single token. /// /// Returns `None` if the string contains zero *or two or more* tokens. /// The token is malformed if the returned error is not `None`. /// /// Beware that unescape errors are not checked at tokenization time. pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> { - lex_first_token(text) - .filter(|(token, _)| token.len == TextSize::of(text)) - .map(|(token, error)| (token.kind, error)) + let (first_token, err) = lex_first_token(text)?; + if first_token.len != TextSize::of(text) { + return None; + } + Some((first_token.kind, err)) } /// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and @@ -79,9 +80,11 @@ pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxEr /// /// Beware that unescape errors are not checked at tokenization time. pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> { - lex_first_token(text) - .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text)) - .map(|(token, _error)| token.kind) + let (single_token, err) = lex_single_syntax_kind(text)?; + if err.is_some() { + return None; + } + Some(single_token) } /// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token |