diff options
36 files changed, 121 insertions, 502 deletions
diff --git a/crates/hir-def/src/body/lower.rs b/crates/hir-def/src/body/lower.rs index 84662aa2778..e8303ec40f7 100644 --- a/crates/hir-def/src/body/lower.rs +++ b/crates/hir-def/src/body/lower.rs @@ -972,7 +972,7 @@ impl From<ast::LiteralKind> for Literal { } } LiteralKind::FloatNumber(lit) => { - let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s)); + let ty = lit.suffix().and_then(BuiltinFloat::from_suffix); Literal::Float(Default::default(), ty) } LiteralKind::ByteString(bs) => { diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 2be6c78a6e7..aa09912f302 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -4,7 +4,10 @@ use base_db::{AnchoredPath, Edition, FileId}; use cfg::CfgExpr; use either::Either; use mbe::{parse_exprs_with_sep, parse_to_token_tree}; -use syntax::{ast, SmolStr}; +use syntax::{ + ast::{self, AstToken}, + SmolStr, +}; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; @@ -355,7 +358,14 @@ fn unreachable_expand( } fn unquote_str(lit: &tt::Literal) -> Option<String> { - let token = ast::make::literal(&lit.to_string()).as_string()?; + let lit = ast::make::tokens::literal(&lit.to_string()); + let token = ast::String::cast(lit)?; + token.value().map(|it| it.into_owned()) +} + +fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> { + let lit = ast::make::tokens::literal(&lit.to_string()); + let token = ast::ByteString::cast(lit)?; token.value().map(|it| it.into_owned()) } @@ -432,16 +442,12 @@ fn concat_bytes_expand( for (i, t) in tt.token_trees.iter().enumerate() { match t { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let lit = ast::make::literal(&lit.to_string()); - match lit.kind() { - ast::LiteralKind::ByteString(s) => { - s.value() - .unwrap_or_default() - .into_iter() - .for_each(|x| bytes.push(x.to_string())); - } - ast::LiteralKind::Byte(_) => { - bytes.push(lit.to_string()); + let token = ast::make::tokens::literal(&lit.to_string()); + match token.kind() { + syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()), + syntax::SyntaxKind::BYTE_STRING => { + let components = unquote_byte_string(lit).unwrap_or_else(Vec::new); + components.into_iter().for_each(|x| bytes.push(x.to_string())); } _ => { err.get_or_insert(mbe::ExpandError::UnexpectedToken.into()); @@ -475,10 +481,10 @@ fn concat_bytes_expand_subtree( for (ti, tt) in tree.token_trees.iter().enumerate() { match tt { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let lit = ast::make::literal(&lit.to_string()); + let lit = ast::make::tokens::literal(&lit.to_string()); match lit.kind() { - ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => { - bytes.push(lit.to_string()); + syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => { + bytes.push(lit.text().to_string()) } _ => { return Err(mbe::ExpandError::UnexpectedToken.into()); diff --git a/crates/hir-ty/src/tests/simple.rs b/crates/hir-ty/src/tests/simple.rs index 4b8a7e782de..d4d61c2167c 100644 --- a/crates/hir-ty/src/tests/simple.rs +++ b/crates/hir-ty/src/tests/simple.rs @@ -2733,14 +2733,3 @@ fn f() { "#, ); } - -#[test] -fn nested_tuple_index() { - check_no_mismatches( - r#" -fn main() { - let fld: i32 = ((0,),).0.0; -} -"#, - ); -} diff --git a/crates/ide-completion/src/completions/dot.rs b/crates/ide-completion/src/completions/dot.rs index 6a553eadc19..9b5ececfd07 100644 --- a/crates/ide-completion/src/completions/dot.rs +++ b/crates/ide-completion/src/completions/dot.rs @@ -793,24 +793,4 @@ fn main() { ", ) } - - #[test] - fn tuple_index_completion() { - check( - r#" -struct I; -impl I { - fn i_method(&self) {} -} -struct S((), I); - -fn f(s: S) { - s.1.$0 -} -"#, - expect![[r#" - me i_method() fn(&self) - "#]], - ); - } } diff --git a/crates/ide-completion/src/completions/postfix.rs b/crates/ide-completion/src/completions/postfix.rs index be0f6748891..28256bd7042 100644 --- a/crates/ide-completion/src/completions/postfix.rs +++ b/crates/ide-completion/src/completions/postfix.rs @@ -5,7 +5,7 @@ mod format_like; use hir::{Documentation, HasAttrs}; use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap}; use syntax::{ - ast::{self, AstNode, LiteralKind}, + ast::{self, AstNode, AstToken}, SyntaxKind::{EXPR_STMT, STMT_LIST}, TextRange, TextSize, }; @@ -194,7 +194,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) { } if let ast::Expr::Literal(literal) = dot_receiver.clone() { - if let LiteralKind::String(literal_text) = literal.kind() { + if let Some(literal_text) = ast::String::cast(literal.token()) { add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text); } } diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index bafbf6d9cc8..e04fd5a7c7f 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -30,15 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { SymbolKind::Field.into() } - INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 - | FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(), - DOT if matches!( - token.prev_token().map(|n| n.kind()), - Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2) - ) => - { - HlTag::NumericLiteral.into() - } + INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(), BYTE => HlTag::ByteLiteral.into(), CHAR => HlTag::CharLiteral.into(), IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index d7d6c79e3d3..60bc2901211 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="numeric_literal">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">></span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span> diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 79fd1250e0e..21a0aa4284a 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -260,31 +260,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), - FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => { - // Reassemble a split-up float token. - let mut range = range; - let mut text = token.to_text(conv).to_string(); - if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 { - let (dot, dot_range) = conv.bump().unwrap(); - text += &*dot.to_text(conv); - range = TextRange::new(range.start(), dot_range.end()); - - if kind == FLOAT_NUMBER_START_2 { - let (tail, tail_range) = conv.bump().unwrap(); - text += &*tail.to_text(conv); - range = TextRange::new(range.start(), tail_range.end()); - } - } - - result.push( - tt::Leaf::from(tt::Literal { - id: conv.id_alloc().alloc(range, synth_id), - text: text.into(), - }) - .into(), - ); - continue; - } k if k.is_literal() => make_leaf!(Literal), LIFETIME_IDENT => { let char_unit = TextSize::of('\''); diff --git a/crates/mbe/src/to_parser_input.rs b/crates/mbe/src/to_parser_input.rs index 958f6433206..6faa147218e 100644 --- a/crates/mbe/src/to_parser_input.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -35,13 +35,15 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input { let is_negated = lit.text.starts_with('-'); let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; - let lexed_str = parser::LexedStr::new(inner_text); - if lexed_str.is_empty() { - panic!("failed to convert literal: {:?}", lit); - } - for i in 0..lexed_str.len() { - res.push(lexed_str.kind(i)); - } + let kind = parser::LexedStr::single_token(inner_text) + .map(|(kind, _error)| kind) + .filter(|kind| { + kind.is_literal() + && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER)) + }) + .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit)); + + res.push(kind); } tt::Leaf::Ident(ident) => match ident.text.as_ref() { "_" => res.push(T![_]), diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 92f33025387..4efbf9a606e 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -39,7 +39,6 @@ mod generic_params; mod types; use crate::{ - grammar::expressions::FLOAT_LITERAL_FIRST, parser::{CompletedMarker, Marker, Parser}, SyntaxKind::{self, *}, TokenSet, T, @@ -319,15 +318,9 @@ fn name_ref(p: &mut Parser) { } fn name_ref_or_index(p: &mut Parser) { - assert!( - p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) - ); + assert!(p.at(IDENT) || p.at(INT_NUMBER)); let m = p.start(); - if p.at_ts(FLOAT_LITERAL_FIRST) { - p.bump_remap(FLOAT_NUMBER_PART); - } else { - p.bump_any(); - } + p.bump_any(); m.complete(p, NAME_REF); } diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index 290083b3434..b063c73a9d6 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -3,7 +3,7 @@ mod atom; use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; +pub(super) use self::atom::{literal, LITERAL_FIRST}; #[derive(PartialEq, Eq)] pub(super) enum Semicolon { @@ -452,9 +452,6 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo(); // y.bar::<T>(1, 2,); -// -// 0e0.sin(); -// 0e0f32.sin(); // } fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); @@ -472,16 +469,17 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo; // x.0.bar; -// x.0. bar; -// x.0.1; // x.0(); // } fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.])); let m = lhs.precede(p); p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { + if p.at(IDENT) || p.at(INT_NUMBER) { name_ref_or_index(p); + } else if p.at(FLOAT_NUMBER) { + // FIXME: How to recover and instead parse INT + T![.]? + p.bump_any(); } else { p.error("expected field name or number"); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 5ab148ff288..10e5d897e07 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -17,58 +17,22 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[ T![true], T![false], INT_NUMBER, - FLOAT_NUMBER_START_0, - FLOAT_NUMBER_START_1, - FLOAT_NUMBER_START_2, + FLOAT_NUMBER, BYTE, CHAR, STRING, BYTE_STRING, ]); -pub(crate) const FLOAT_LITERAL_FIRST: TokenSet = - TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]); - pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> { if !p.at_ts(LITERAL_FIRST) { return None; } let m = p.start(); - if p.at_ts(FLOAT_LITERAL_FIRST) { - float_literal(p); - } else { - // Everything else is just one token. - p.bump_any(); - } + p.bump_any(); Some(m.complete(p, LITERAL)) } -// test float_literal -// fn f() { -// 0.0; -// 1.; -// 0e0; -// 0e0f32; -// 1.23f64; -// } -pub(crate) fn float_literal(p: &mut Parser) { - // Floats can be up to 3 tokens. The first token indicates how many there are. - let f = p.start(); - if p.at(FLOAT_NUMBER_START_0) { - p.bump(FLOAT_NUMBER_START_0); - } else if p.at(FLOAT_NUMBER_START_1) { - p.bump(FLOAT_NUMBER_START_1); - p.bump(DOT); - } else if p.at(FLOAT_NUMBER_START_2) { - p.bump(FLOAT_NUMBER_START_2); - p.bump(DOT); - p.bump(FLOAT_NUMBER_PART); - } else { - unreachable!(); - } - f.complete(p, FLOAT_LITERAL); -} - // E.g. for after the break in `if break {}`, this should not match pub(super) const ATOM_EXPR_FIRST: TokenSet = LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[ diff --git a/crates/parser/src/grammar/patterns.rs b/crates/parser/src/grammar/patterns.rs index c16bd8d0c7a..1f622b32e5b 100644 --- a/crates/parser/src/grammar/patterns.rs +++ b/crates/parser/src/grammar/patterns.rs @@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> { } fn is_literal_pat_start(p: &Parser) -> bool { - p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART) + p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) || p.at_ts(expressions::LITERAL_FIRST) } diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index 9887960871f..f4b9988eacb 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -177,7 +177,7 @@ impl<'a> Converter<'a> { rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::Literal { kind, .. } => { - self.extend_literal(token_text, kind); + self.extend_literal(token_text.len(), kind); return; } @@ -223,7 +223,7 @@ impl<'a> Converter<'a> { self.push(syntax_kind, token_text.len(), err); } - fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) { + fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) { let mut err = ""; let syntax_kind = match *kind { @@ -237,27 +237,7 @@ impl<'a> Converter<'a> { if empty_exponent { err = "Missing digits after the exponent symbol"; } - - // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0` - // is lexed as a float, we split floats that contain a `.` into 3 tokens. - // To ensure that later stages can always reconstruct the token correctly, the first - // token in the sequence indicates the number of following tokens that are part of - // the float literal. - if let Some((before, after)) = token_text.split_once('.') { - let err = if err.is_empty() { None } else { Some(err) }; - - assert!(!before.is_empty()); - let tok = - if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 }; - self.push(tok, before.len(), None); - self.push(DOT, 1, None); - if !after.is_empty() { - self.push(FLOAT_NUMBER_PART, after.len(), err); - } - return; - } - - FLOAT_NUMBER_START_0 + FLOAT_NUMBER } rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { @@ -315,6 +295,6 @@ impl<'a> Converter<'a> { }; let err = if err.is_empty() { None } else { Some(err) }; - self.push(syntax_kind, token_text.len(), err); + self.push(syntax_kind, len, err); } } diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 5ddcd6ad6a4..628fa745e75 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -110,10 +110,7 @@ pub enum SyntaxKind { RAW_KW, MACRO_RULES_KW, INT_NUMBER, - FLOAT_NUMBER_START_0, - FLOAT_NUMBER_START_1, - FLOAT_NUMBER_START_2, - FLOAT_NUMBER_PART, + FLOAT_NUMBER, CHAR, BYTE, STRING, @@ -230,7 +227,6 @@ pub enum SyntaxKind { PATH, PATH_SEGMENT, LITERAL, - FLOAT_LITERAL, RENAME, VISIBILITY, WHERE_CLAUSE, @@ -290,8 +286,7 @@ impl SyntaxKind { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 - | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } @@ -391,5 +386,5 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; [float_number_start_0] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_0 } ; [float_number_start_1] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_1 } ; [float_number_start_2] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_2 } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; } pub use T; diff --git a/crates/parser/test_data/lexer/err/empty_exponent.rast b/crates/parser/test_data/lexer/err/empty_exponent.rast index 73de4cac243..af03d73ced9 100644 --- a/crates/parser/test_data/lexer/err/empty_exponent.rast +++ b/crates/parser/test_data/lexer/err/empty_exponent.rast @@ -1,14 +1,14 @@ -FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol +FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol +FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" INT_NUMBER "42" DOT "." @@ -30,35 +30,19 @@ DOT "." IDENT "E" MINUS "-" WHITESPACE "\n\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" diff --git a/crates/parser/test_data/lexer/ok/numbers.rast b/crates/parser/test_data/lexer/ok/numbers.rast index 428bdf8a1f9..8d13c3f6106 100644 --- a/crates/parser/test_data/lexer/ok/numbers.rast +++ b/crates/parser/test_data/lexer/ok/numbers.rast @@ -4,8 +4,7 @@ INT_NUMBER "00" WHITESPACE " " INT_NUMBER "0_" WHITESPACE " " -FLOAT_NUMBER_START_1 "0" -DOT "." +FLOAT_NUMBER "0." WHITESPACE " " INT_NUMBER "0z" WHITESPACE "\n" @@ -21,13 +20,11 @@ INT_NUMBER "001279" WHITESPACE " " INT_NUMBER "0_1279" WHITESPACE " " -FLOAT_NUMBER_START_2 "0" -DOT "." -FLOAT_NUMBER_PART "1279" +FLOAT_NUMBER "0.1279" WHITESPACE " " -FLOAT_NUMBER_START_0 "0e1279" +FLOAT_NUMBER "0e1279" WHITESPACE " " -FLOAT_NUMBER_START_0 "0E1279" +FLOAT_NUMBER "0E1279" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -40,7 +37,7 @@ IDENT "foo" L_PAREN "(" R_PAREN ")" WHITESPACE "\n" -FLOAT_NUMBER_START_0 "0e+1" +FLOAT_NUMBER "0e+1" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -48,19 +45,13 @@ IDENT "e" PLUS "+" INT_NUMBER "1" WHITESPACE "\n" -FLOAT_NUMBER_START_2 "0" -DOT "." -FLOAT_NUMBER_PART "0E-2" +FLOAT_NUMBER "0.0E-2" WHITESPACE "\n" -FLOAT_NUMBER_START_2 "0___0" -DOT "." -FLOAT_NUMBER_PART "10000____0000e+111__" +FLOAT_NUMBER "0___0.10000____0000e+111__" WHITESPACE "\n" INT_NUMBER "1i64" WHITESPACE " " -FLOAT_NUMBER_START_2 "92" -DOT "." -FLOAT_NUMBER_PART "0f32" +FLOAT_NUMBER "92.0f32" WHITESPACE " " INT_NUMBER "11__s" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast index 70cd030da1e..4064a7a1ff2 100644 --- a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast +++ b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast @@ -32,9 +32,7 @@ SOURCE_FILE INT_NUMBER "1" COMMA "," WHITESPACE " " - FLOAT_NUMBER_START_2 "2" - DOT "." - FLOAT_NUMBER_PART "0" + FLOAT_NUMBER "2.0" WHITESPACE "\n " R_CURLY "}" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast index a1efb3a9fb2..8498724b9ef 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast @@ -41,39 +41,6 @@ SOURCE_FILE SEMICOLON ";" WHITESPACE "\n " EXPR_STMT - FIELD_EXPR - FIELD_EXPR - PATH_EXPR - PATH - PATH_SEGMENT - NAME_REF - IDENT "x" - DOT "." - NAME_REF - FLOAT_NUMBER_PART "0" - DOT "." - WHITESPACE " " - NAME_REF - IDENT "bar" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - FIELD_EXPR - FIELD_EXPR - PATH_EXPR - PATH - PATH_SEGMENT - NAME_REF - IDENT "x" - DOT "." - NAME_REF - FLOAT_NUMBER_PART "0" - DOT "." - NAME_REF - FLOAT_NUMBER_PART "1" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT CALL_EXPR FIELD_EXPR PATH_EXPR diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs index 551b1ecaf02..b8da2ddc309 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs @@ -1,7 +1,5 @@ fn foo() { x.foo; x.0.bar; - x.0. bar; - x.0.1; x.0(); } diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast index b3236976b98..403c265ea35 100644 --- a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast @@ -57,10 +57,7 @@ SOURCE_FILE EQ "=" WHITESPACE " " LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "2" - DOT "." - FLOAT_NUMBER_PART "0" + FLOAT_NUMBER "2.0" SEMICOLON ";" WHITESPACE "\n " LET_STMT diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast index 69f1055b7e3..dcbcfe1231e 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast @@ -58,32 +58,6 @@ SOURCE_FILE COMMA "," R_PAREN ")" SEMICOLON ";" - WHITESPACE "\n\n " - EXPR_STMT - METHOD_CALL_EXPR - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0" - DOT "." - NAME_REF - IDENT "sin" - ARG_LIST - L_PAREN "(" - R_PAREN ")" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - METHOD_CALL_EXPR - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0f32" - DOT "." - NAME_REF - IDENT "sin" - ARG_LIST - L_PAREN "(" - R_PAREN ")" - SEMICOLON ";" WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs index 3e5d464e238..1a3aa35ae8e 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs @@ -1,7 +1,4 @@ fn foo() { x.foo(); y.bar::<T>(1, 2,); - - 0e0.sin(); - 0e0f32.sin(); } diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast deleted file mode 100644 index df4fb6eb417..00000000000 --- a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast +++ /dev/null @@ -1,51 +0,0 @@ -SOURCE_FILE - FN - FN_KW "fn" - WHITESPACE " " - NAME - IDENT "f" - PARAM_LIST - L_PAREN "(" - R_PAREN ")" - WHITESPACE " " - BLOCK_EXPR - STMT_LIST - L_CURLY "{" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "0" - DOT "." - FLOAT_NUMBER_PART "0" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_1 "1" - DOT "." - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0f32" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "1" - DOT "." - FLOAT_NUMBER_PART "23f64" - SEMICOLON ";" - WHITESPACE "\n" - R_CURLY "}" - WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs deleted file mode 100644 index 0d51ec1252f..00000000000 --- a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn f() { - 0.0; - 1.; - 0e0; - 0e0f32; - 1.23f64; -} diff --git a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast index b2f66e2f1e7..55ce31275fb 100644 --- a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast +++ b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast @@ -19,10 +19,7 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "1" - DOT "." - FLOAT_NUMBER_PART "0f32" + FLOAT_NUMBER "1.0f32" DOT "." NAME_REF IDENT "floor" @@ -43,10 +40,7 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "1" - DOT "." - FLOAT_NUMBER_PART "0f32" + FLOAT_NUMBER "1.0f32" DOT "." NAME_REF IDENT "floor" diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 85270dee624..62aa4783994 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -365,20 +365,13 @@ MacroExpr = Literal = Attr* value:( - 'int_number' | FloatLiteral + 'int_number' | 'float_number' | 'string' | 'raw_string' | 'byte_string' | 'raw_byte_string' | 'true' | 'false' | 'char' | 'byte' ) -FloatLiteral = - 'float_number_start_0'? - 'float_number_start_1'? - 'float_number_start_2'? - '.'? - 'float_number_part'? - PathExpr = Attr* Path diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index 11d81d47783..17785152bc5 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -8,7 +8,7 @@ use crate::{ operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp}, support, AstChildren, AstNode, }, - AstToken, SyntaxElement, + AstToken, SyntaxKind::*, SyntaxNode, SyntaxToken, T, }; @@ -282,32 +282,30 @@ pub enum LiteralKind { String(ast::String), ByteString(ast::ByteString), IntNumber(ast::IntNumber), - FloatNumber(ast::FloatLiteral), + FloatNumber(ast::FloatNumber), Char(ast::Char), Byte(ast::Byte), Bool(bool), } impl ast::Literal { - pub fn value(&self) -> SyntaxElement { + pub fn token(&self) -> SyntaxToken { self.syntax() .children_with_tokens() .find(|e| e.kind() != ATTR && !e.kind().is_trivia()) + .and_then(|e| e.into_token()) .unwrap() } + pub fn kind(&self) -> LiteralKind { - let token = match self.value() { - rowan::NodeOrToken::Node(node) => { - return LiteralKind::FloatNumber( - ast::FloatLiteral::cast(node).expect("unreachable"), - ); - } - rowan::NodeOrToken::Token(token) => token, - }; + let token = self.token(); if let Some(t) = ast::IntNumber::cast(token.clone()) { return LiteralKind::IntNumber(t); } + if let Some(t) = ast::FloatNumber::cast(token.clone()) { + return LiteralKind::FloatNumber(t); + } if let Some(t) = ast::String::cast(token.clone()) { return LiteralKind::String(t); } @@ -327,26 +325,6 @@ impl ast::Literal { _ => unreachable!(), } } - - pub fn as_string(&self) -> Option<ast::String> { - match self.kind() { - LiteralKind::String(it) => Some(it), - _ => None, - } - } - - pub fn as_byte_string(&self) -> Option<ast::ByteString> { - match self.kind() { - LiteralKind::ByteString(it) => Some(it), - _ => None, - } - } -} - -impl ast::FloatLiteral { - pub fn suffix(&self) -> Option<String> { - ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) - } } pub enum BlockModifier { @@ -386,7 +364,7 @@ impl ast::BlockExpr { fn test_literal_with_attr() { let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#); let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap(); - assert_eq!(lit.value().to_string(), r#""Hello""#); + assert_eq!(lit.token().text(), r#""Hello""#); } impl ast::RecordExprField { diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index 9d5af8e63ca..cf90ba64cff 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1086,26 +1086,6 @@ impl UnderscoreExpr { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FloatLiteral { - pub(crate) syntax: SyntaxNode, -} -impl FloatLiteral { - pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_start_0]) - } - pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_start_1]) - } - pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_start_2]) - } - pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) } - pub fn float_number_part_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_part]) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct StmtList { pub(crate) syntax: SyntaxNode, } @@ -2739,17 +2719,6 @@ impl AstNode for UnderscoreExpr { } fn syntax(&self) -> &SyntaxNode { &self.syntax } } -impl AstNode for FloatLiteral { - fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL } - fn cast(syntax: SyntaxNode) -> Option<Self> { - if Self::can_cast(syntax.kind()) { - Some(Self { syntax }) - } else { - None - } - } - fn syntax(&self) -> &SyntaxNode { &self.syntax } -} impl AstNode for StmtList { fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST } fn cast(syntax: SyntaxNode) -> Option<Self> { @@ -4639,11 +4608,6 @@ impl std::fmt::Display for UnderscoreExpr { std::fmt::Display::fmt(self.syntax(), f) } } -impl std::fmt::Display for FloatLiteral { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } -} impl std::fmt::Display for StmtList { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs index 1e1a55e3269..a3209c5abd2 100644 --- a/crates/syntax/src/ast/generated/tokens.rs +++ b/crates/syntax/src/ast/generated/tokens.rs @@ -112,16 +112,16 @@ impl AstToken for IntNumber { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FloatNumberPart { +pub struct FloatNumber { pub(crate) syntax: SyntaxToken, } -impl std::fmt::Display for FloatNumberPart { +impl std::fmt::Display for FloatNumber { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.syntax, f) } } -impl AstToken for FloatNumberPart { - fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART } +impl AstToken for FloatNumber { + fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER } fn cast(syntax: SyntaxToken) -> Option<Self> { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) diff --git a/crates/syntax/src/ast/make.rs b/crates/syntax/src/ast/make.rs index 846f4f3c71a..5908dda8e63 100644 --- a/crates/syntax/src/ast/make.rs +++ b/crates/syntax/src/ast/make.rs @@ -799,11 +799,6 @@ pub fn struct_( )) } -pub fn literal(text: &str) -> ast::Literal { - assert_eq!(text.trim(), text); - ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)) -} - #[track_caller] fn ast_from_text<N: AstNode>(text: &str) -> N { let parse = SourceFile::parse(text); @@ -832,7 +827,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken { pub mod tokens { use once_cell::sync::Lazy; - use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; + use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| { SourceFile::parse( @@ -863,6 +858,12 @@ pub mod tokens { sf.syntax().first_child_or_token().unwrap().into_token().unwrap() } + pub fn literal(text: &str) -> SyntaxToken { + assert_eq!(text.trim(), text); + let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)); + lit.syntax().first_child_or_token().unwrap().into_token().unwrap() + } + pub fn single_newline() -> SyntaxToken { let res = SOURCE_FILE .tree() diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs index 5b19b5ed2c2..f2153ca9211 100644 --- a/crates/syntax/src/ast/node_ext.rs +++ b/crates/syntax/src/ast/node_ext.rs @@ -555,9 +555,7 @@ impl ast::FieldExpr { self.syntax .children_with_tokens() // FIXME: Accepting floats here to reject them in validation later - .find(|c| { - c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART - }) + .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER) .as_ref() .and_then(SyntaxElement::as_token) .cloned() diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 5f2e7231d93..4b6dc236b53 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -321,7 +321,7 @@ impl ast::IntNumber { } } -impl ast::FloatNumberPart { +impl ast::FloatNumber { pub fn suffix(&self) -> Option<&str> { let text = self.text(); let mut indices = text.char_indices(); @@ -355,24 +355,14 @@ impl Radix { #[cfg(test)] mod tests { - use crate::ast::{self, make}; + use crate::ast::{self, make, FloatNumber, IntNumber}; fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { - let suffix = match make::literal(lit).kind() { - ast::LiteralKind::FloatNumber(f) => f.suffix(), - // `1f32` lexes as an INT_NUMBER - ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()), - e => unreachable!("{e:?}"), - }; - assert_eq!(suffix.as_deref(), expected.into()); + assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); } fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { - let i = match make::literal(lit).kind() { - ast::LiteralKind::IntNumber(i) => i, - _ => unreachable!(), - }; - assert_eq!(i.suffix(), expected.into()); + assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); } #[test] @@ -400,11 +390,12 @@ mod tests { } fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { - let s = match make::literal(&format!("\"{}\"", lit)).kind() { - ast::LiteralKind::String(s) => s, - _ => unreachable!(), - }; - assert_eq!(s.value().as_deref(), expected.into()); + assert_eq!( + ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) } + .value() + .as_deref(), + expected.into() + ); } #[test] diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index f5a78e4119d..2f6932a1add 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -71,17 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], - literals: &[ - "INT_NUMBER", - "FLOAT_NUMBER_START_0", - "FLOAT_NUMBER_START_1", - "FLOAT_NUMBER_START_2", - "FLOAT_NUMBER_PART", - "CHAR", - "BYTE", - "STRING", - "BYTE_STRING", - ], + literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", @@ -193,7 +183,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "PATH", "PATH_SEGMENT", "LITERAL", - "FLOAT_LITERAL", "RENAME", "VISIBILITY", "WHERE_CLAUSE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index 33bf2c0cbad..4cfb8075cb1 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -462,10 +462,6 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; - [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART }; - [float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 }; - [float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 }; - [float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 }; } pub use T; }; @@ -589,7 +585,7 @@ impl Field { fn lower(grammar: &Grammar) -> AstSrc { let mut res = AstSrc { - tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident" + tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident" .split_ascii_whitespace() .map(|it| it.to_string()) .collect::<Vec<_>>(), diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs index 3edca3eb8f9..c2c2c82e11f 100644 --- a/crates/syntax/src/validation.rs +++ b/crates/syntax/src/validation.rs @@ -119,15 +119,8 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) { text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end)) } - let token = literal.value(); - let text; - let text = match &token { - rowan::NodeOrToken::Node(node) => { - text = node.text().to_string(); - &*text - } - rowan::NodeOrToken::Token(token) => token.text(), - }; + let token = literal.token(); + let text = token.text(); // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205) let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| { |