diff options
| author | Lukas Wirth <lukastw97@gmail.com> | 2024-07-16 09:59:39 +0200 |
|---|---|---|
| committer | Lukas Wirth <lukastw97@gmail.com> | 2024-07-16 10:11:59 +0200 |
| commit | 7f8a54bbee4fec4a6885c7554a2864a62bf40638 (patch) | |
| tree | 2c94ef3a76c5c8b74747000eb63a8aeb43c50fd9 | |
| parent | a2c46b96b818a0a0b12ef9e3b520dd6196d24afd (diff) | |
| download | rust-7f8a54bbee4fec4a6885c7554a2864a62bf40638.tar.gz rust-7f8a54bbee4fec4a6885c7554a2864a62bf40638.zip | |
Switch token trees to use Symbols
51 files changed, 593 insertions, 399 deletions
diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock index 500a150b57b..241392edb1e 100644 --- a/src/tools/rust-analyzer/Cargo.lock +++ b/src/tools/rust-analyzer/Cargo.lock @@ -149,6 +149,7 @@ dependencies = [ "mbe", "oorandom", "rustc-hash", + "smol_str", "syntax", "tt", ] @@ -1045,6 +1046,7 @@ version = "0.0.0" dependencies = [ "arrayvec", "cov-mark", + "intern", "parser", "rustc-hash", "smallvec", @@ -1324,8 +1326,8 @@ version = "0.0.0" dependencies = [ "base-db", "indexmap", + "intern", "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "mbe", "paths", "rustc-hash", "serde", @@ -1343,6 +1345,7 @@ version = "0.0.0" dependencies = [ "base-db", "expect-test", + "intern", "libloading", "mbe", "memmap2", @@ -1966,6 +1969,7 @@ dependencies = [ "base-db", "cfg", "hir-expand", + "intern", "rustc-hash", "span", "stdx", @@ -2218,8 +2222,8 @@ name = "tt" version = "0.0.0" dependencies = [ "arrayvec", + "intern", "ra-ap-rustc_lexer", - "smol_str", "stdx", "text-size", ] diff --git a/src/tools/rust-analyzer/crates/cfg/Cargo.toml b/src/tools/rust-analyzer/crates/cfg/Cargo.toml index 9b3a5026ac8..784e86649d1 100644 --- a/src/tools/rust-analyzer/crates/cfg/Cargo.toml +++ b/src/tools/rust-analyzer/crates/cfg/Cargo.toml @@ -16,6 +16,7 @@ rustc-hash.workspace = true # locals deps tt.workspace = true +smol_str.workspace = true [dev-dependencies] expect-test = "1.4.1" diff --git a/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs b/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs index b7dbb7b5fdd..9c95f0e4e4b 100644 --- a/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs +++ b/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs @@ -4,7 +4,7 @@ use std::{fmt, slice::Iter as SliceIter}; -use tt::SmolStr; +use smol_str::SmolStr; /// A simple configuration value passed in from the outside. #[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] @@ -66,7 +66,7 @@ impl CfgExpr { fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> { let name = match it.next() { None => return None, - Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(), + Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(), Some(_) => return Some(CfgExpr::Invalid), }; @@ -77,10 +77,9 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => { it.next(); it.next(); - // FIXME: escape? raw string? - let value = - SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"')); - CfgAtom::KeyValue { key: name, value }.into() + // FIXME: escape? + let value = literal.symbol.as_str().into(); + CfgAtom::KeyValue { key: name.as_str().into(), value }.into() } _ => return Some(CfgExpr::Invalid), } @@ -96,7 +95,7 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> _ => CfgExpr::Invalid, } } - _ => CfgAtom::Flag(name).into(), + _ => CfgAtom::Flag(name.as_str().into()).into(), }; // Eat comma separator diff --git a/src/tools/rust-analyzer/crates/cfg/src/lib.rs b/src/tools/rust-analyzer/crates/cfg/src/lib.rs index 8b30286a0a8..5ef7a104dda 100644 --- a/src/tools/rust-analyzer/crates/cfg/src/lib.rs +++ b/src/tools/rust-analyzer/crates/cfg/src/lib.rs @@ -8,10 +8,10 @@ mod tests; use std::fmt; use rustc_hash::FxHashSet; -use tt::SmolStr; pub use cfg_expr::{CfgAtom, CfgExpr}; pub use dnf::DnfExpr; +use smol_str::SmolStr; /// Configuration options used for conditional compilation on items with `cfg` attributes. /// We have two kind of options in different namespaces: atomic options like `unix`, and diff --git a/src/tools/rust-analyzer/crates/hir-def/src/attr.rs b/src/tools/rust-analyzer/crates/hir-def/src/attr.rs index 5bd60c58e9b..1a5ac96aa29 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/attr.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/attr.rs @@ -159,14 +159,14 @@ impl Attrs { pub fn has_doc_hidden(&self) -> bool { self.by_key("doc").tt_values().any(|tt| { tt.delimiter.kind == DelimiterKind::Parenthesis && - matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden") + matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::hidden) }) } pub fn has_doc_notable_trait(&self) -> bool { self.by_key("doc").tt_values().any(|tt| { tt.delimiter.kind == DelimiterKind::Parenthesis && - matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "notable_trait") + matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::notable_trait) }) } @@ -267,7 +267,7 @@ impl DocExpr { fn next_doc_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<DocExpr> { let name = match it.next() { None => return None, - Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(), + Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(), Some(_) => return Some(DocExpr::Invalid), }; @@ -275,13 +275,16 @@ fn next_doc_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<DocExpr> let ret = match it.as_slice().first() { Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => { match it.as_slice().get(1) { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => { + Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { + symbol: text, + kind: tt::LitKind::Str, + .. + }))) => { it.next(); it.next(); // FIXME: escape? raw string? - let value = - SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"')); - DocAtom::KeyValue { key: name, value }.into() + let value = SmolStr::new(text.as_str()); + DocAtom::KeyValue { key: name.as_str().into(), value }.into() } _ => return Some(DocExpr::Invalid), } @@ -294,7 +297,7 @@ fn next_doc_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<DocExpr> _ => DocExpr::Invalid, } } - _ => DocAtom::Flag(name).into(), + _ => DocAtom::Flag(name.as_str().into()).into(), }; // Eat comma separator @@ -311,10 +314,11 @@ fn parse_comma_sep<S>(subtree: &tt::Subtree<S>) -> Vec<SmolStr> { .token_trees .iter() .filter_map(|tt| match tt { - tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - // FIXME: escape? raw string? - Some(SmolStr::new(lit.text.trim_start_matches('"').trim_end_matches('"'))) - } + tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { + kind: tt::LitKind::Str, + symbol: text, + .. + })) => Some(SmolStr::new(text.as_str())), _ => None, }) .collect() @@ -598,14 +602,14 @@ impl<'attr> AttrQuery<'attr> { /// #[doc(html_root_url = "url")] /// ^^^^^^^^^^^^^ key /// ``` - pub fn find_string_value_in_tt(self, key: &'attr str) -> Option<&SmolStr> { + pub fn find_string_value_in_tt(self, key: &'attr str) -> Option<&str> { self.tt_values().find_map(|tt| { let name = tt.token_trees.iter() - .skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text, ..} )) if text == key)) + .skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { sym, ..} )) if sym.as_str() == key)) .nth(2); match name { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text), + Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text.as_str()), _ => None } }) diff --git a/src/tools/rust-analyzer/crates/hir-def/src/builtin_type.rs b/src/tools/rust-analyzer/crates/hir-def/src/builtin_type.rs index 6dc1c4546e8..14b9af84e6f 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/builtin_type.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/builtin_type.rs @@ -6,7 +6,7 @@ use std::fmt; use hir_expand::name::{AsName, Name}; -use intern::sym; +use intern::{sym, Symbol}; /// Different signed int types. #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum BuiltinInt { @@ -143,6 +143,18 @@ impl BuiltinInt { }; Some(res) } + pub fn from_suffix_sym(suffix: &Symbol) -> Option<BuiltinInt> { + let res = match suffix { + s if *s == sym::isize => Self::Isize, + s if *s == sym::i8 => Self::I8, + s if *s == sym::i16 => Self::I16, + s if *s == sym::i32 => Self::I32, + s if *s == sym::i64 => Self::I64, + s if *s == sym::i128 => Self::I128, + _ => return None, + }; + Some(res) + } } #[rustfmt::skip] @@ -160,6 +172,19 @@ impl BuiltinUint { }; Some(res) } + pub fn from_suffix_sym(suffix: &Symbol) -> Option<BuiltinUint> { + let res = match suffix { + s if *s == sym::usize => Self::Usize, + s if *s == sym::u8 => Self::U8, + s if *s == sym::u16 => Self::U16, + s if *s == sym::u32 => Self::U32, + s if *s == sym::u64 => Self::U64, + s if *s == sym::u128 => Self::U128, + + _ => return None, + }; + Some(res) + } } #[rustfmt::skip] diff --git a/src/tools/rust-analyzer/crates/hir-def/src/data.rs b/src/tools/rust-analyzer/crates/hir-def/src/data.rs index 964a5f04bfd..e2bb02c0c13 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/data.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/data.rs @@ -150,7 +150,7 @@ fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> { let mut indices = Vec::new(); for args in tt.token_trees.chunks(2) { match &args[0] { - tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => match lit.text.parse() { + tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => match lit.symbol.as_str().parse() { Ok(index) => indices.push(index), Err(_) => break, }, diff --git a/src/tools/rust-analyzer/crates/hir-def/src/data/adt.rs b/src/tools/rust-analyzer/crates/hir-def/src/data/adt.rs index 0fe73418e51..3942c2a98af 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/data/adt.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/data/adt.rs @@ -9,7 +9,7 @@ use hir_expand::{ name::{AsName, Name}, HirFileId, InFile, }; -use intern::Interned; +use intern::{sym, Interned}; use la_arena::Arena; use rustc_abi::{Align, Integer, IntegerType, ReprFlags, ReprOptions}; use syntax::ast::{self, HasName, HasVisibility}; @@ -112,12 +112,12 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> { let mut tts = tt.token_trees.iter().peekable(); while let Some(tt) = tts.next() { if let TokenTree::Leaf(Leaf::Ident(ident)) = tt { - flags.insert(match &*ident.text { - "packed" => { + flags.insert(match &ident.sym { + s if *s == sym::packed => { let pack = if let Some(TokenTree::Subtree(tt)) = tts.peek() { tts.next(); if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() { - lit.text.parse().unwrap_or_default() + lit.symbol.as_str().parse().unwrap_or_default() } else { 0 } @@ -129,11 +129,11 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> { Some(if let Some(min_pack) = min_pack { min_pack.min(pack) } else { pack }); ReprFlags::empty() } - "align" => { + s if *s == sym::align => { if let Some(TokenTree::Subtree(tt)) = tts.peek() { tts.next(); if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() { - if let Ok(align) = lit.text.parse() { + if let Ok(align) = lit.symbol.as_str().parse() { let align = Align::from_bytes(align).ok(); max_align = max_align.max(align); } @@ -141,13 +141,13 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> { } ReprFlags::empty() } - "C" => ReprFlags::IS_C, - "transparent" => ReprFlags::IS_TRANSPARENT, - "simd" => ReprFlags::IS_SIMD, + s if *s == sym::C => ReprFlags::IS_C, + s if *s == sym::transparent => ReprFlags::IS_TRANSPARENT, + s if *s == sym::simd => ReprFlags::IS_SIMD, repr => { - if let Some(builtin) = BuiltinInt::from_suffix(repr) + if let Some(builtin) = BuiltinInt::from_suffix_sym(repr) .map(Either::Left) - .or_else(|| BuiltinUint::from_suffix(repr).map(Either::Right)) + .or_else(|| BuiltinUint::from_suffix_sym(repr).map(Either::Right)) { int = Some(match builtin { Either::Left(bi) => match bi { diff --git a/src/tools/rust-analyzer/crates/hir-def/src/db.rs b/src/tools/rust-analyzer/crates/hir-def/src/db.rs index eac8f0dd74a..1f728a0b38b 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/db.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/db.rs @@ -278,7 +278,7 @@ fn crate_supports_no_std(db: &dyn DefDatabase, crate_id: CrateId) -> bool { tt.split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(p)) if p.char == ',')); for output in segments.skip(1) { match output { - [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "no_std" => { + [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::no_std => { return true } _ => {} diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs index b0543727c27..6861e0f2f5b 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs @@ -63,6 +63,7 @@ use base_db::{CrateId, FileId}; use hir_expand::{ name::Name, proc_macro::ProcMacroKind, ErasedAstId, HirFileId, InFile, MacroCallId, MacroDefId, }; +use intern::Symbol; use itertools::Itertools; use la_arena::Arena; use rustc_hash::{FxHashMap, FxHashSet}; @@ -148,11 +149,11 @@ struct DefMapCrateData { proc_macro_loading_error: Option<Box<str>>, /// Custom attributes registered with `#![register_attr]`. - registered_attrs: Vec<SmolStr>, + registered_attrs: Vec<Symbol>, /// Custom tool modules registered with `#![register_tool]`. - registered_tools: Vec<SmolStr>, + registered_tools: Vec<Symbol>, /// Unstable features of Rust enabled with `#![feature(A, B)]`. - unstable_features: FxHashSet<SmolStr>, + unstable_features: FxHashSet<Symbol>, /// #[rustc_coherence_is_core] rustc_coherence_is_core: bool, no_core: bool, @@ -170,7 +171,7 @@ impl DefMapCrateData { fn_proc_macro_mapping: FxHashMap::default(), proc_macro_loading_error: None, registered_attrs: Vec::new(), - registered_tools: PREDEFINED_TOOLS.into(), + registered_tools: PREDEFINED_TOOLS.iter().map(|it| Symbol::intern(it)).collect(), unstable_features: FxHashSet::default(), rustc_coherence_is_core: false, no_core: false, @@ -447,15 +448,15 @@ impl DefMap { self.derive_helpers_in_scope.get(&id.map(|it| it.upcast())).map(Deref::deref) } - pub fn registered_tools(&self) -> &[SmolStr] { + pub fn registered_tools(&self) -> &[Symbol] { &self.data.registered_tools } - pub fn registered_attrs(&self) -> &[SmolStr] { + pub fn registered_attrs(&self) -> &[Symbol] { &self.data.registered_attrs } - pub fn is_unstable_feature_enabled(&self, feature: &str) -> bool { + pub fn is_unstable_feature_enabled(&self, feature: &Symbol) -> bool { self.data.unstable_features.contains(feature) } diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs index f842027d642..747860fd8e1 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs @@ -7,7 +7,7 @@ use hir_expand::{ MacroCallId, MacroCallKind, MacroDefId, }; use span::SyntaxContextId; -use syntax::{ast, SmolStr}; +use syntax::ast; use triomphe::Arc; use crate::{ @@ -79,20 +79,20 @@ impl DefMap { let segments = path.segments(); if let Some(name) = segments.first() { - let name = name.to_smol_str(); - let pred = |n: &_| *n == name; + let name = name.symbol(); + let pred = |n: &_| *n == *name; - let is_tool = self.data.registered_tools.iter().map(SmolStr::as_str).any(pred); + let is_tool = self.data.registered_tools.iter().any(pred); // FIXME: tool modules can be shadowed by actual modules if is_tool { return true; } if segments.len() == 1 { - if find_builtin_attr_idx(&name).is_some() { + if find_builtin_attr_idx(name).is_some() { return true; } - if self.data.registered_attrs.iter().map(SmolStr::as_str).any(pred) { + if self.data.registered_attrs.iter().any(pred) { return true; } } diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs index 13abcce78ea..a614d99de6c 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs @@ -317,20 +317,20 @@ impl DefCollector<'_> { .into_iter() .flatten() .filter_map(|(feat, _)| match feat.segments() { - [name] => Some(name.to_smol_str()), + [name] => Some(name.symbol().clone()), _ => None, }); crate_data.unstable_features.extend(features); } () if *attr_name == sym::register_attr.clone() => { if let Some(ident) = attr.single_ident_value() { - crate_data.registered_attrs.push(ident.text.clone()); + crate_data.registered_attrs.push(ident.sym.clone()); cov_mark::hit!(register_attr); } } () if *attr_name == sym::register_tool.clone() => { if let Some(ident) = attr.single_ident_value() { - crate_data.registered_tools.push(ident.text.clone()); + crate_data.registered_tools.push(ident.sym.clone()); cov_mark::hit!(register_tool); } } @@ -2129,9 +2129,7 @@ impl ModCollector<'_, '_> { let is_export = export_attr.exists(); let local_inner = if is_export { export_attr.tt_values().flat_map(|it| it.token_trees.iter()).any(|it| match it { - tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { - ident.text.contains("local_inner_macros") - } + tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => ident.sym == sym::local_inner_macros, _ => false, }) } else { diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs index 5052708dc93..4789b1f0145 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs @@ -1,6 +1,7 @@ //! Nameres-specific procedural macro data and helpers. use hir_expand::name::{AsName, Name}; +use intern::sym; use crate::attr::Attrs; use crate::tt::{Leaf, TokenTree}; @@ -67,7 +68,7 @@ pub(crate) fn parse_macro_name_and_helper_attrs(tt: &[TokenTree]) -> Option<(Nam TokenTree::Leaf(Leaf::Punct(comma)), TokenTree::Leaf(Leaf::Ident(attributes)), TokenTree::Subtree(helpers) - ] if comma.char == ',' && attributes.text == "attributes" => + ] if comma.char == ',' && attributes.sym == sym::attributes => { let helpers = helpers .token_trees diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs index 49a104fa118..8ef7b7049ae 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs @@ -62,7 +62,7 @@ impl RawAttrs { Attr { id, input: Some(Box::new(AttrInput::Literal(tt::Literal { - text, + symbol: text, span, kind, suffix: None, @@ -243,7 +243,7 @@ impl Attr { let span = span_map.span_for_range(range); let input = if let Some(ast::Expr::Literal(lit)) = ast.expr() { let token = lit.token(); - Some(Box::new(AttrInput::Literal(token_to_literal(token.text().into(), span)))) + Some(Box::new(AttrInput::Literal(token_to_literal(token.text(), span)))) } else if let Some(tt) = ast.token_tree() { let tree = syntax_node_to_token_tree( tt.syntax(), @@ -260,8 +260,8 @@ impl Attr { fn from_tt(db: &dyn ExpandDatabase, mut tt: &[tt::TokenTree], id: AttrId) -> Option<Attr> { if matches!(tt, - [tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text, .. })), ..] - if text == "unsafe" + [tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { sym, .. })), ..] + if *sym == sym::unsafe_ ) { match tt.get(1) { Some(tt::TokenTree::Subtree(subtree)) => tt = &subtree.token_trees, @@ -313,10 +313,10 @@ impl Attr { pub fn string_value(&self) -> Option<&str> { match self.input.as_deref()? { AttrInput::Literal(tt::Literal { - text, + symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. - }) => Some(text), + }) => Some(text.as_str()), _ => None, } } @@ -324,23 +324,24 @@ impl Attr { /// #[path = "string"] pub fn string_value_with_span(&self) -> Option<(&str, span::Span)> { match self.input.as_deref()? { - AttrInput::Literal(it) => match it.text.strip_prefix('r') { - Some(it) => it.trim_matches('#'), - None => it.text.as_str(), - } - .strip_prefix('"')? - .strip_suffix('"') - .zip(Some(it.span)), + AttrInput::Literal(tt::Literal { + symbol: text, + kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), + span, + suffix: _, + }) => Some((text.as_str(), *span)), _ => None, } } pub fn string_value_unescape(&self) -> Option<Cow<'_, str>> { match self.input.as_deref()? { - AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::StrRaw(_), .. }) => { - Some(Cow::Borrowed(text)) + AttrInput::Literal(tt::Literal { + symbol: text, kind: tt::LitKind::StrRaw(_), .. + }) => Some(Cow::Borrowed(text.as_str())), + AttrInput::Literal(tt::Literal { symbol: text, kind: tt::LitKind::Str, .. }) => { + unescape(text.as_str()) } - AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::Str, .. }) => unescape(text), _ => None, } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs index 180d8f05627..b2924ae6736 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs @@ -82,7 +82,7 @@ enum VariantShape { } fn tuple_field_iterator(span: Span, n: usize) -> impl Iterator<Item = tt::Ident> { - (0..n).map(move |it| tt::Ident::new(format!("f{it}"), span)) + (0..n).map(move |it| tt::Ident::new(&format!("f{it}"), span)) } impl VariantShape { @@ -693,14 +693,14 @@ fn partial_eq_expand(span: Span, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> } [first, rest @ ..] => { let rest = rest.iter().map(|it| { - let t1 = tt::Ident::new(format!("{}_self", it.text), it.span); - let t2 = tt::Ident::new(format!("{}_other", it.text), it.span); + let t1 = tt::Ident::new(&format!("{}_self", it.sym), it.span); + let t2 = tt::Ident::new(&format!("{}_other", it.sym), it.span); let and_and = and_and(span); quote!(span =>#and_and #t1 .eq( #t2 )) }); let first = { - let t1 = tt::Ident::new(format!("{}_self", first.text), first.span); - let t2 = tt::Ident::new(format!("{}_other", first.text), first.span); + let t1 = tt::Ident::new(&format!("{}_self", first.sym), first.span); + let t2 = tt::Ident::new(&format!("{}_other", first.sym), first.span); quote!(span =>#t1 .eq( #t2 )) }; quote!(span =>#first ##rest) @@ -730,7 +730,7 @@ fn self_and_other_patterns( let self_patterns = adt.shape.as_pattern_map( name, |it| { - let t = tt::Ident::new(format!("{}_self", it.text), it.span); + let t = tt::Ident::new(&format!("{}_self", it.sym), it.span); quote!(span =>#t) }, span, @@ -738,7 +738,7 @@ fn self_and_other_patterns( let other_patterns = adt.shape.as_pattern_map( name, |it| { - let t = tt::Ident::new(format!("{}_other", it.text), it.span); + let t = tt::Ident::new(&format!("{}_other", it.sym), it.span); quote!(span =>#t) }, span, @@ -776,8 +776,8 @@ fn ord_expand(span: Span, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> { |(pat1, pat2, fields)| { let mut body = quote!(span =>#krate::cmp::Ordering::Equal); for f in fields.into_iter().rev() { - let t1 = tt::Ident::new(format!("{}_self", f.text), f.span); - let t2 = tt::Ident::new(format!("{}_other", f.text), f.span); + let t1 = tt::Ident::new(&format!("{}_self", f.sym), f.span); + let t2 = tt::Ident::new(&format!("{}_other", f.sym), f.span); body = compare(krate, quote!(span =>#t1), quote!(span =>#t2), body, span); } let fat_arrow = fat_arrow(span); @@ -838,8 +838,8 @@ fn partial_ord_expand(span: Span, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> let mut body = quote!(span =>#krate::option::Option::Some(#krate::cmp::Ordering::Equal)); for f in fields.into_iter().rev() { - let t1 = tt::Ident::new(format!("{}_self", f.text), f.span); - let t2 = tt::Ident::new(format!("{}_other", f.text), f.span); + let t1 = tt::Ident::new(&format!("{}_self", f.sym), f.span); + let t2 = tt::Ident::new(&format!("{}_other", f.sym), f.span); body = compare(krate, quote!(span =>#t1), quote!(span =>#t2), body, span); } let fat_arrow = fat_arrow(span); diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs index fb12adb87cb..4b1a3d69728 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs @@ -1,10 +1,9 @@ //! Builtin macro -use ::tt::SmolStr; use base_db::{AnchoredPath, FileId}; use cfg::CfgExpr; use either::Either; -use intern::sym; +use intern::{sym, Symbol}; use mbe::{parse_exprs_with_sep, parse_to_token_tree}; use span::{Edition, Span, SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}; use stdx::format_to; @@ -181,10 +180,10 @@ fn line_expand( ExpandResult::ok(tt::Subtree { delimiter: tt::Delimiter::invisible_spanned(span), token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text: "0".into(), + symbol: sym::INTEGER_0.clone(), span, kind: tt::LitKind::Integer, - suffix: Some(Box::new("u32".into())), + suffix: Some(sym::u32.clone()), }))]), }) } @@ -301,12 +300,12 @@ fn format_args_nl_expand( let mut tt = tt.clone(); tt.delimiter.kind = tt::DelimiterKind::Parenthesis; if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text, + symbol: text, kind: tt::LitKind::Str, .. }))) = tt.token_trees.first_mut() { - *text = format_smolstr!("{text}\\n"); + *text = Symbol::intern(&format_smolstr!("{}\\n", text.as_str())); } ExpandResult::ok(quote! {span => builtin #pound format_args #tt @@ -460,14 +459,14 @@ fn compile_error_expand( ) -> ExpandResult<tt::Subtree> { let err = match &*tt.token_trees { [tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text, + symbol: text, span: _, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), suffix: _, }))] => // FIXME: Use the span here! { - ExpandError::other(Box::from(&*unescape_str(text))) + ExpandError::other(Box::from(unescape_str(text).as_str())) } _ => ExpandError::other("`compile_error!` argument must be a string"), }; @@ -507,18 +506,20 @@ fn concat_expand( // as-is. match it.kind { tt::LitKind::Char => { - if let Ok(c) = unescape_char(&it.text) { + if let Ok(c) = unescape_char(it.symbol.as_str()) { text.extend(c.escape_default()); } record_span(it.span); } - tt::LitKind::Integer | tt::LitKind::Float => format_to!(text, "{}", it.text), + tt::LitKind::Integer | tt::LitKind::Float => { + format_to!(text, "{}", it.symbol.as_str()) + } tt::LitKind::Str => { - text.push_str(&it.text); + text.push_str(it.symbol.as_str()); record_span(it.span); } tt::LitKind::StrRaw(_) => { - format_to!(text, "{}", it.text.escape_debug()); + format_to!(text, "{}", it.symbol.as_str().escape_debug()); record_span(it.span); } tt::LitKind::Byte @@ -531,9 +532,9 @@ fn concat_expand( } // handle boolean literals tt::TokenTree::Leaf(tt::Leaf::Ident(id)) - if i % 2 == 0 && (id.text == "true" || id.text == "false") => + if i % 2 == 0 && (id.sym == sym::true_ || id.sym == sym::false_) => { - text.push_str(id.text.as_str()); + text.push_str(id.sym.as_str()); record_span(id.span); } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (), @@ -562,21 +563,21 @@ fn concat_bytes_expand( }; for (i, t) in tt.token_trees.iter().enumerate() { match t { - tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text, span, kind, suffix: _ })) => { + tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { symbol: text, span, kind, suffix: _ })) => { record_span(*span); match kind { tt::LitKind::Byte => { - if let Ok(b) = unescape_byte(text) { + if let Ok(b) = unescape_byte(text.as_str()) { bytes.extend( b.escape_ascii().filter_map(|it| char::from_u32(it as u32)), ); } } tt::LitKind::ByteStr => { - bytes.push_str(text); + bytes.push_str(text.as_str()); } tt::LitKind::ByteStrRaw(_) => { - bytes.extend(text.escape_debug()); + bytes.extend(text.as_str().escape_debug()); } _ => { err.get_or_insert(mbe::ExpandError::UnexpectedToken.into()); @@ -602,7 +603,7 @@ fn concat_bytes_expand( value: tt::Subtree { delimiter: tt::Delimiter::invisible_spanned(span), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text: bytes.into(), + symbol: Symbol::intern(&bytes), span, kind: tt::LitKind::ByteStr, suffix: None, @@ -621,24 +622,24 @@ fn concat_bytes_expand_subtree( for (ti, tt) in tree.token_trees.iter().enumerate() { match tt { tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text, + symbol: text, span, kind: tt::LitKind::Byte, suffix: _, })) => { - if let Ok(b) = unescape_byte(text) { + if let Ok(b) = unescape_byte(text.as_str()) { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } record_span(*span); } tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text, + symbol: text, span, kind: tt::LitKind::Integer, suffix: _, })) => { record_span(*span); - if let Ok(b) = text.parse::<u8>() { + if let Ok(b) = text.as_str().parse::<u8>() { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } } @@ -662,7 +663,7 @@ fn concat_idents_expand( for (i, t) in tt.token_trees.iter().enumerate() { match t { tt::TokenTree::Leaf(tt::Leaf::Ident(id)) => { - ident.push_str(id.text.as_str()); + ident.push_str(id.sym.as_str()); } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (), _ => { @@ -671,7 +672,7 @@ fn concat_idents_expand( } } // FIXME merge spans - let ident = tt::Ident { text: ident.into(), span, is_raw: tt::IdentIsRaw::No }; + let ident = tt::Ident { sym: Symbol::intern(&ident), span, is_raw: tt::IdentIsRaw::No }; ExpandResult { value: quote!(span =>#ident), err } } @@ -694,12 +695,12 @@ fn relative_file( } } -fn parse_string(tt: &tt::Subtree) -> Result<(SmolStr, Span), ExpandError> { +fn parse_string(tt: &tt::Subtree) -> Result<(Symbol, Span), ExpandError> { tt.token_trees .first() .and_then(|tt| match tt { tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text, + symbol: text, span, kind: tt::LitKind::Str, suffix: _, @@ -739,7 +740,7 @@ pub fn include_input_to_file_id( arg_id: MacroCallId, arg: &tt::Subtree, ) -> Result<FileId, ExpandError> { - relative_file(db, arg_id, &parse_string(arg)?.0, false) + relative_file(db, arg_id, parse_string(arg)?.0.as_str(), false) } fn include_bytes_expand( @@ -752,7 +753,7 @@ fn include_bytes_expand( let res = tt::Subtree { delimiter: tt::Delimiter::invisible_spanned(span), token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - text: r#"b"""#.into(), + symbol: Symbol::empty(), span, kind: tt::LitKind::ByteStrRaw(1), suffix: None, @@ -778,7 +779,7 @@ fn include_str_expand( // it's unusual to `include_str!` a Rust file), but we can return an empty string. // Ideally, we'd be able to offer a precise expansion if the user asks for macro // expansion. - let file_id = match relative_file(db, arg_id, &path, true) { + let file_id = match relative_file(db, arg_id, path.as_str(), true) { Ok(file_id) => file_id, Err(_) => { return ExpandResult::ok(quote!(span =>"")); @@ -791,9 +792,9 @@ fn include_str_expand( ExpandResult::ok(quote!(span =>#text)) } -fn get_env_inner(db: &dyn ExpandDatabase, arg_id: MacroCallId, key: &str) -> Option<String> { +fn get_env_inner(db: &dyn ExpandDatabase, arg_id: MacroCallId, key: &Symbol) -> Option<String> { let krate = db.lookup_intern_macro_call(arg_id).krate; - db.crate_graph()[krate].env.get(key).map(|it| it.escape_debug().to_string()) + db.crate_graph()[krate].env.get(key.as_str()).map(|it| it.escape_debug().to_string()) } fn env_expand( @@ -813,7 +814,7 @@ fn env_expand( let s = get_env_inner(db, arg_id, &key).unwrap_or_else(|| { // The only variable rust-analyzer ever sets is `OUT_DIR`, so only diagnose that to avoid // unnecessary diagnostics for eg. `CARGO_PKG_NAME`. - if key == "OUT_DIR" { + if key.as_str() == "OUT_DIR" { err = Some(ExpandError::other(r#"`OUT_DIR` not set, enable "build scripts" to fix"#)); } @@ -867,15 +868,16 @@ fn quote_expand( ) } -fn unescape_str(s: &SmolStr) -> SmolStr { - if s.contains('\\') { +fn unescape_str(s: &Symbol) -> Symbol { + if s.as_str().contains('\\') { + let s = s.as_str(); let mut buf = String::with_capacity(s.len()); unescape_unicode(s, Mode::Str, &mut |_, c| { if let Ok(c) = c { buf.push(c) } }); - buf.into() + Symbol::intern(&buf) } else { s.clone() } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs b/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs index 55ae19068f9..5f038cfe687 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs @@ -9,7 +9,6 @@ use syntax::{ AstNode, NodeOrToken, SyntaxElement, SyntaxKind, SyntaxNode, T, }; use tracing::{debug, warn}; -use tt::SmolStr; use crate::{db::ExpandDatabase, proc_macro::ProcMacroKind, MacroCallLoc, MacroDefKind}; @@ -263,7 +262,7 @@ where let name = match iter.next() { None => return None, Some(NodeOrToken::Token(element)) => match element.kind() { - syntax::T![ident] => SmolStr::new(element.text()), + syntax::T![ident] => element.text().to_owned(), _ => return Some(CfgExpr::Invalid), }, Some(_) => return Some(CfgExpr::Invalid), @@ -302,13 +301,13 @@ where if (value_token.kind() == syntax::SyntaxKind::STRING) => { let value = value_token.text(); - let value = SmolStr::new(value.trim_matches('"')); - Some(CfgExpr::Atom(CfgAtom::KeyValue { key: name, value })) + let value = value.trim_matches('"').into(); + Some(CfgExpr::Atom(CfgAtom::KeyValue { key: name.into(), value })) } _ => None, } } - _ => Some(CfgExpr::Atom(CfgAtom::Flag(name))), + _ => Some(CfgExpr::Atom(CfgAtom::Flag(name.into()))), }, }; if let Some(NodeOrToken::Token(element)) = iter.peek() { diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs b/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs index 2e7865ed3bf..3d3df833618 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs @@ -120,10 +120,10 @@ impl DeclarativeMacroExpander { .token_tree_value()? .token_trees { - [tt::TokenTree::Leaf(tt::Leaf::Ident(i)), ..] => match &*i.text { - "transparent" => Some(Transparency::Transparent), - "semitransparent" => Some(Transparency::SemiTransparent), - "opaque" => Some(Transparency::Opaque), + [tt::TokenTree::Leaf(tt::Leaf::Ident(i)), ..] => match &i.sym { + s if *s == sym::transparent => Some(Transparency::Transparent), + s if *s == sym::semitransparent => Some(Transparency::SemiTransparent), + s if *s == sym::opaque => Some(Transparency::Opaque), _ => None, }, _ => None, diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs index 2896afed084..cf51535630f 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs @@ -1,6 +1,7 @@ //! To make attribute macros work reliably when typing, we need to take care to //! fix up syntax errors in the code we're passing to them. +use intern::sym; use mbe::DocCommentDesugarMode; use rustc_hash::{FxHashMap, FxHashSet}; use smallvec::SmallVec; @@ -80,7 +81,7 @@ pub(crate) fn fixup_syntax( original.push(original_tree); let span = span_map.span_for_range(node_range); let replacement = Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: Span { range: TextRange::new(TextSize::new(idx), FIXUP_DUMMY_RANGE_END), anchor: SpanAnchor { ast_id: FIXUP_DUMMY_AST_ID, ..span.anchor }, @@ -100,7 +101,7 @@ pub(crate) fn fixup_syntax( // incomplete field access: some_expr.| append.insert(node.clone().into(), vec![ Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }), @@ -138,7 +139,7 @@ pub(crate) fn fixup_syntax( }; append.insert(if_token.into(), vec![ Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }), @@ -169,7 +170,7 @@ pub(crate) fn fixup_syntax( }; append.insert(while_token.into(), vec![ Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }), @@ -217,7 +218,7 @@ pub(crate) fn fixup_syntax( }; append.insert(match_token.into(), vec![ Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }), @@ -247,12 +248,12 @@ pub(crate) fn fixup_syntax( }; let [pat, in_token, iter] = [ - "_", - "in", - "__ra_fixup" - ].map(|text| + sym::underscore.clone(), + sym::in_.clone(), + sym::__ra_fixup.clone(), + ].map(|sym| Leaf::Ident(Ident { - text: text.into(), + sym, span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }), @@ -286,7 +287,7 @@ pub(crate) fn fixup_syntax( if it.name_ref().is_some() && it.expr().is_none() { append.insert(colon.into(), vec![ Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }) @@ -299,7 +300,7 @@ pub(crate) fn fixup_syntax( if it.segment().is_none() { append.insert(colon.into(), vec![ Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }) @@ -333,7 +334,7 @@ pub(crate) fn fixup_syntax( if it.body().is_none() { append.insert(node.into(), vec![ Leaf::Ident(Ident { - text: "__ra_fixup".into(), + sym: sym::__ra_fixup.clone(), span: fake_span(node_range), is_raw: tt::IdentIsRaw::No }) @@ -448,9 +449,9 @@ mod tests { // `TokenTree`s, see the last assertion in `check()`. fn check_leaf_eq(a: &tt::Leaf, b: &tt::Leaf) -> bool { match (a, b) { - (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.text == b.text, + (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.symbol == b.symbol, (tt::Leaf::Punct(a), tt::Leaf::Punct(b)) => a.char == b.char, - (tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.text == b.text, + (tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.sym == b.sym, _ => false, } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/inert_attr_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/inert_attr_macro.rs index 0c112554e1f..ee15b1b5ce9 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/inert_attr_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/inert_attr_macro.rs @@ -10,6 +10,7 @@ use std::sync::OnceLock; +use intern::Symbol; use rustc_hash::FxHashMap; pub struct BuiltinAttribute { @@ -26,11 +27,16 @@ pub struct AttributeTemplate { pub name_value_str: Option<&'static str>, } -pub fn find_builtin_attr_idx(name: &str) -> Option<usize> { - static BUILTIN_LOOKUP_TABLE: OnceLock<FxHashMap<&'static str, usize>> = OnceLock::new(); +pub fn find_builtin_attr_idx(name: &Symbol) -> Option<usize> { + static BUILTIN_LOOKUP_TABLE: OnceLock<FxHashMap<Symbol, usize>> = OnceLock::new(); BUILTIN_LOOKUP_TABLE .get_or_init(|| { - INERT_ATTRIBUTES.iter().map(|attr| attr.name).enumerate().map(|(a, b)| (b, a)).collect() + INERT_ATTRIBUTES + .iter() + .map(|attr| attr.name) + .enumerate() + .map(|(a, b)| (Symbol::intern(b), a)) + .collect() }) .get(name) .copied() diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs b/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs index 907e939153b..2c26fe414d9 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs @@ -316,30 +316,36 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: &[tt::TokenTree]) -> Option<ModP tt::Leaf::Punct(tt::Punct { char: ':', .. }) => PathKind::Abs, _ => return None, }, - tt::Leaf::Ident(tt::Ident { text, span, .. }) if text == "$crate" => { + tt::Leaf::Ident(tt::Ident { sym: text, span, .. }) if *text == sym::dollar_crate => { resolve_crate_root(db, span.ctx).map(PathKind::DollarCrate).unwrap_or(PathKind::Crate) } - tt::Leaf::Ident(tt::Ident { text, .. }) if text == "self" => PathKind::SELF, - tt::Leaf::Ident(tt::Ident { text, .. }) if text == "super" => { + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::self_ => PathKind::SELF, + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::super_ => { let mut deg = 1; - while let Some(tt::Leaf::Ident(tt::Ident { text, span, is_raw })) = leaves.next() { - if text != "super" { - segments.push(Name::new(text, *is_raw, span.ctx)); + while let Some(tt::Leaf::Ident(tt::Ident { sym: text, span, is_raw })) = leaves.next() { + if *text != sym::super_ { + segments.push(Name::new_symbol_maybe_raw(text.clone(), *is_raw, span.ctx)); break; } deg += 1; } PathKind::Super(deg) } - tt::Leaf::Ident(tt::Ident { text, .. }) if text == "crate" => PathKind::Crate, + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::crate_ => PathKind::Crate, tt::Leaf::Ident(ident) => { - segments.push(Name::new(&ident.text, ident.is_raw, ident.span.ctx)); + segments.push(Name::new_symbol_maybe_raw( + ident.sym.clone(), + ident.is_raw, + ident.span.ctx, + )); PathKind::Plain } _ => return None, }; segments.extend(leaves.filter_map(|leaf| match leaf { - ::tt::Leaf::Ident(ident) => Some(Name::new(&ident.text, ident.is_raw, ident.span.ctx)), + ::tt::Leaf::Ident(ident) => { + Some(Name::new_symbol_maybe_raw(ident.sym.clone(), ident.is_raw, ident.span.ctx)) + } _ => None, })); Some(ModPath { kind, segments }) diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/name.rs b/src/tools/rust-analyzer/crates/hir-expand/src/name.rs index fce9df6722b..64c094bd287 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/name.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/name.rs @@ -17,6 +17,8 @@ use syntax::{ast, format_smolstr, utils::is_raw_identifier, SmolStr}; pub struct Name { symbol: Symbol, ctx: (), + // FIXME: We should probably encode rawness as a property here instead, once we have hygiene + // in here we've got 4 bytes of padding to fill anyways } impl fmt::Debug for Name { @@ -187,14 +189,22 @@ impl Name { &self.symbol } - pub const fn new_symbol(doc: Symbol, ctx: SyntaxContextId) -> Self { + pub const fn new_symbol(symbol: Symbol, ctx: SyntaxContextId) -> Self { _ = ctx; - Self { symbol: doc, ctx: () } + Self { symbol, ctx: () } + } + + pub fn new_symbol_maybe_raw(sym: Symbol, raw: tt::IdentIsRaw, ctx: SyntaxContextId) -> Self { + if raw.no() { + Self { symbol: sym, ctx: () } + } else { + Name::new(sym.as_str(), raw, ctx) + } } // FIXME: This needs to go once we have hygiene - pub const fn new_symbol_root(doc: Symbol) -> Self { - Self { symbol: doc, ctx: () } + pub const fn new_symbol_root(sym: Symbol) -> Self { + Self { symbol: sym, ctx: () } } } @@ -250,7 +260,7 @@ impl AsName for ast::NameOrNameRef { impl<Span> AsName for tt::Ident<Span> { fn as_name(&self) -> Name { - Name::resolve(&self.text) + Name::resolve(self.sym.as_str()) } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs b/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs index f1d28450b31..a65d46161e5 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs @@ -1,14 +1,14 @@ //! A simplified version of quote-crate like quasi quote macro #![allow(clippy::crate_in_macro_def)] -use intern::Symbol; +use intern::{sym, Symbol}; use span::Span; use tt::IdentIsRaw; use crate::name::Name; -pub(crate) const fn dollar_crate(span: Span) -> tt::Ident<Span> { - tt::Ident { text: syntax::SmolStr::new_static("$crate"), span, is_raw: tt::IdentIsRaw::No } +pub(crate) fn dollar_crate(span: Span) -> tt::Ident<Span> { + tt::Ident { sym: sym::dollar_crate.clone(), span, is_raw: tt::IdentIsRaw::No } } // A helper macro quote macro @@ -99,7 +99,7 @@ macro_rules! __quote { ($span:ident $tt:ident ) => { vec![ { crate::tt::Leaf::Ident(crate::tt::Ident { - text: stringify!($tt).into(), + sym: intern::Symbol::intern(stringify!($tt)), span: $span, is_raw: tt::IdentIsRaw::No, }).into() @@ -177,12 +177,6 @@ impl ToTokenTree for crate::tt::TokenTree { } } -impl ToTokenTree for &crate::tt::TokenTree { - fn to_token(self, _: Span) -> crate::tt::TokenTree { - self.clone() - } -} - impl ToTokenTree for crate::tt::Subtree { fn to_token(self, _: Span) -> crate::tt::TokenTree { self.into() @@ -198,35 +192,34 @@ macro_rules! impl_to_to_tokentrees { leaf.into() } } - - impl ToTokenTree for &$ty { - fn to_token($this, $span: Span) -> crate::tt::TokenTree { - let leaf: crate::tt::Leaf = $im.clone().into(); - leaf.into() - } - } )* } } +impl<T: ToTokenTree + Clone> ToTokenTree for &T { + fn to_token(self, span: Span) -> crate::tt::TokenTree { + self.clone().to_token(span) + } +} + impl_to_to_tokentrees! { - span: u32 => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer, suffix: None } }; - span: usize => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer, suffix: None } }; - span: i32 => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer, suffix: None } }; - span: bool => self { crate::tt::Ident{text: self.to_string().into(), span, is_raw: tt::IdentIsRaw::No } }; + span: u32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } }; + span: usize => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } }; + span: i32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } }; + span: bool => self { crate::tt::Ident{sym: if self { sym::true_.clone() } else { sym::false_.clone() }, span, is_raw: tt::IdentIsRaw::No } }; _span: crate::tt::Leaf => self { self }; _span: crate::tt::Literal => self { self }; _span: crate::tt::Ident => self { self }; _span: crate::tt::Punct => self { self }; - span: &str => self { crate::tt::Literal{text: (*self).into(), span, kind: tt::LitKind::Str, suffix: None }}; - span: String => self { crate::tt::Literal{text: self.into(), span, kind: tt::LitKind::Str, suffix: None }}; + span: &str => self { crate::tt::Literal{symbol: Symbol::intern(self), span, kind: tt::LitKind::Str, suffix: None }}; + span: String => self { crate::tt::Literal{symbol: Symbol::intern(&self), span, kind: tt::LitKind::Str, suffix: None }}; span: Name => self { let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str()); - crate::tt::Ident{text: s.into(), span, is_raw } + crate::tt::Ident{sym: Symbol::intern(s), span, is_raw } }; span: Symbol => self { let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str()); - crate::tt::Ident{text: s.into(), span, is_raw } + crate::tt::Ident{sym: Symbol::intern(s), span, is_raw } }; } @@ -236,6 +229,7 @@ mod tests { use ::tt::IdentIsRaw; use base_db::FileId; use expect_test::expect; + use intern::Symbol; use span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}; use syntax::{TextRange, TextSize}; @@ -268,7 +262,7 @@ mod tests { fn mk_ident(name: &str) -> crate::tt::Ident { let (is_raw, s) = IdentIsRaw::split_from_symbol(name); - crate::tt::Ident { text: s.into(), span: DUMMY, is_raw } + crate::tt::Ident { sym: Symbol::intern(s), span: DUMMY, is_raw } } #[test] diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/diagnostics/match_check/pat_analysis.rs b/src/tools/rust-analyzer/crates/hir-ty/src/diagnostics/match_check/pat_analysis.rs index 01e43a67e43..1374d0c38b8 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/diagnostics/match_check/pat_analysis.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/diagnostics/match_check/pat_analysis.rs @@ -3,6 +3,7 @@ use std::fmt; use hir_def::{DefWithBodyId, EnumId, EnumVariantId, HasModule, LocalFieldId, ModuleId, VariantId}; +use intern::sym; use once_cell::unsync::Lazy; use rustc_index::IndexVec; use rustc_pattern_analysis::{ @@ -75,9 +76,9 @@ pub(crate) struct MatchCheckCtx<'db> { impl<'db> MatchCheckCtx<'db> { pub(crate) fn new(module: ModuleId, body: DefWithBodyId, db: &'db dyn HirDatabase) -> Self { let def_map = db.crate_def_map(module.krate()); - let exhaustive_patterns = def_map.is_unstable_feature_enabled("exhaustive_patterns"); + let exhaustive_patterns = def_map.is_unstable_feature_enabled(&sym::exhaustive_patterns); let min_exhaustive_patterns = - def_map.is_unstable_feature_enabled("min_exhaustive_patterns"); + def_map.is_unstable_feature_enabled(&sym::min_exhaustive_patterns); Self { module, body, db, exhaustive_patterns, min_exhaustive_patterns } } diff --git a/src/tools/rust-analyzer/crates/hir/src/lib.rs b/src/tools/rust-analyzer/crates/hir/src/lib.rs index c868357ff98..9f33c506708 100644 --- a/src/tools/rust-analyzer/crates/hir/src/lib.rs +++ b/src/tools/rust-analyzer/crates/hir/src/lib.rs @@ -3380,23 +3380,27 @@ impl BuiltinAttr { if let builtin @ Some(_) = Self::builtin(name) { return builtin; } - let idx = - db.crate_def_map(krate.id).registered_attrs().iter().position(|it| it == name)? as u32; + let idx = db + .crate_def_map(krate.id) + .registered_attrs() + .iter() + .position(|it| it.as_str() == name)? as u32; Some(BuiltinAttr { krate: Some(krate.id), idx }) } fn builtin(name: &str) -> Option<Self> { - hir_expand::inert_attr_macro::find_builtin_attr_idx(name) + hir_expand::inert_attr_macro::find_builtin_attr_idx(&Symbol::intern(name)) .map(|idx| BuiltinAttr { krate: None, idx: idx as u32 }) } - pub fn name(&self, db: &dyn HirDatabase) -> SmolStr { - // FIXME: Return a `Name` here + pub fn name(&self, db: &dyn HirDatabase) -> Name { match self.krate { - Some(krate) => db.crate_def_map(krate).registered_attrs()[self.idx as usize].clone(), - None => { - SmolStr::new(hir_expand::inert_attr_macro::INERT_ATTRIBUTES[self.idx as usize].name) - } + Some(krate) => Name::new_symbol_root( + db.crate_def_map(krate).registered_attrs()[self.idx as usize].clone(), + ), + None => Name::new_symbol_root(Symbol::intern( + hir_expand::inert_attr_macro::INERT_ATTRIBUTES[self.idx as usize].name, + )), } } @@ -3420,13 +3424,15 @@ impl ToolModule { pub(crate) fn by_name(db: &dyn HirDatabase, krate: Crate, name: &str) -> Option<Self> { let krate = krate.id; let idx = - db.crate_def_map(krate).registered_tools().iter().position(|it| it == name)? as u32; + db.crate_def_map(krate).registered_tools().iter().position(|it| it.as_str() == name)? + as u32; Some(ToolModule { krate, idx }) } - pub fn name(&self, db: &dyn HirDatabase) -> SmolStr { - // FIXME: Return a `Name` here - db.crate_def_map(self.krate).registered_tools()[self.idx as usize].clone() + pub fn name(&self, db: &dyn HirDatabase) -> Name { + Name::new_symbol_root( + db.crate_def_map(self.krate).registered_tools()[self.idx as usize].clone(), + ) } } diff --git a/src/tools/rust-analyzer/crates/ide-db/src/defs.rs b/src/tools/rust-analyzer/crates/ide-db/src/defs.rs index dbb2adcd301..a53c156b5da 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/defs.rs +++ b/src/tools/rust-analyzer/crates/ide-db/src/defs.rs @@ -192,13 +192,13 @@ impl Definition { let AttributeTemplate { word, list, name_value_str } = it.template(db)?; let mut docs = "Valid forms are:".to_owned(); if word { - format_to!(docs, "\n - #\\[{}]", name); + format_to!(docs, "\n - #\\[{}]", name.display(db)); } if let Some(list) = list { - format_to!(docs, "\n - #\\[{}({})]", name, list); + format_to!(docs, "\n - #\\[{}({})]", name.display(db), list); } if let Some(name_value_str) = name_value_str { - format_to!(docs, "\n - #\\[{} = {}]", name, name_value_str); + format_to!(docs, "\n - #\\[{} = {}]", name.display(db), name_value_str); } Some(Documentation::new(docs.replace('*', "\\*"))) } @@ -256,8 +256,8 @@ impl Definition { Definition::GenericParam(it) => it.display(db).to_string(), Definition::Label(it) => it.name(db).display(db).to_string(), Definition::ExternCrateDecl(it) => it.display(db).to_string(), - Definition::BuiltinAttr(it) => format!("#[{}]", it.name(db)), - Definition::ToolModule(it) => it.name(db).to_string(), + Definition::BuiltinAttr(it) => format!("#[{}]", it.name(db).display(db)), + Definition::ToolModule(it) => it.name(db).display(db).to_string(), Definition::DeriveHelper(it) => format!("derive_helper {}", it.name(db).display(db)), } } diff --git a/src/tools/rust-analyzer/crates/intern/src/symbol.rs b/src/tools/rust-analyzer/crates/intern/src/symbol.rs index a3cc5c3d6a3..9e3f6d842f3 100644 --- a/src/tools/rust-analyzer/crates/intern/src/symbol.rs +++ b/src/tools/rust-analyzer/crates/intern/src/symbol.rs @@ -174,6 +174,32 @@ impl Symbol { } } + pub fn integer(i: usize) -> Self { + match i { + 0 => symbols::INTEGER_0.clone(), + 1 => symbols::INTEGER_1.clone(), + 2 => symbols::INTEGER_2.clone(), + 3 => symbols::INTEGER_3.clone(), + 4 => symbols::INTEGER_4.clone(), + 5 => symbols::INTEGER_5.clone(), + 6 => symbols::INTEGER_6.clone(), + 7 => symbols::INTEGER_7.clone(), + 8 => symbols::INTEGER_8.clone(), + 9 => symbols::INTEGER_9.clone(), + 10 => symbols::INTEGER_10.clone(), + 11 => symbols::INTEGER_11.clone(), + 12 => symbols::INTEGER_12.clone(), + 13 => symbols::INTEGER_13.clone(), + 14 => symbols::INTEGER_14.clone(), + 15 => symbols::INTEGER_15.clone(), + i => Symbol::intern(&format!("{i}")), + } + } + + pub fn empty() -> Self { + symbols::__empty.clone() + } + pub fn as_str(&self) -> &str { self.repr.as_str() } diff --git a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs index 064335471e9..61b9d099e06 100644 --- a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs +++ b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs @@ -56,6 +56,10 @@ macro_rules! define_symbols { define_symbols! { @WITH_NAME: + __empty = "", + unsafe_ = "unsafe", + in_ = "in", + super_ = "super", self_ = "self", Self_ = "Self", tick_static = "'static", @@ -78,10 +82,18 @@ define_symbols! { INTEGER_14 = "14", INTEGER_15 = "15", fn_ = "fn", + crate_ = "crate", + underscore = "_", + true_ = "true", + false_ = "false", + let_ = "let", + const_ = "const", @PLAIN: + __ra_fixup, add_assign, add, + attributes, align_offset, alloc_layout, alloc, @@ -92,6 +104,9 @@ define_symbols! { bench, bitand_assign, bitand, + notable_trait, + hidden, + local_inner_macros, bitor_assign, bitor, bitxor_assign, @@ -225,9 +240,12 @@ define_symbols! { log_syntax, lt, macro_rules, + ignore, + count, manually_drop, maybe_uninit, metadata_type, + missing, module_path, mul_assign, mul, @@ -349,6 +367,10 @@ define_symbols! { u8, Unknown, unpin, + simd, + C, + align, + packed, unreachable_2015, unreachable_2021, unreachable, @@ -356,5 +378,7 @@ define_symbols! { unsize, usize, v1, + exhaustive_patterns, + min_exhaustive_patterns, va_list } diff --git a/src/tools/rust-analyzer/crates/mbe/Cargo.toml b/src/tools/rust-analyzer/crates/mbe/Cargo.toml index 1002de2104a..57834623e84 100644 --- a/src/tools/rust-analyzer/crates/mbe/Cargo.toml +++ b/src/tools/rust-analyzer/crates/mbe/Cargo.toml @@ -24,6 +24,7 @@ parser.workspace = true tt.workspace = true stdx.workspace = true span.workspace = true +intern.workspace = true [dev-dependencies] test-utils.workspace = true diff --git a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs index 6a2f1c23686..04e78a08334 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs @@ -1,10 +1,11 @@ //! This module add real world mbe example for benchmark tests +use intern::Symbol; use rustc_hash::FxHashMap; use span::{Edition, Span}; use syntax::{ ast::{self, HasName}, - AstNode, SmolStr, + AstNode, }; use test_utils::{bench, bench_fixture, skip_slow_tests}; @@ -228,7 +229,7 @@ fn invocation_fixtures( fn make_ident(ident: &str) -> tt::TokenTree<Span> { tt::Leaf::Ident(tt::Ident { span: DUMMY, - text: SmolStr::new(ident), + sym: Symbol::intern(ident), is_raw: tt::IdentIsRaw::No, }) .into() @@ -239,7 +240,7 @@ fn invocation_fixtures( fn make_literal(lit: &str) -> tt::TokenTree<Span> { tt::Leaf::Literal(tt::Literal { span: DUMMY, - text: SmolStr::new(lit), + symbol: Symbol::intern(lit), kind: tt::LitKind::Str, suffix: None, }) diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander.rs b/src/tools/rust-analyzer/crates/mbe/src/expander.rs index cfad8bcc0b4..55b89aa848d 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander.rs @@ -5,9 +5,9 @@ mod matcher; mod transcriber; +use intern::Symbol; use rustc_hash::FxHashMap; use span::{Edition, Span}; -use syntax::SmolStr; use crate::{parser::MetaVarKind, ExpandError, ExpandResult, MatchedArmIndex}; @@ -110,12 +110,12 @@ pub(crate) fn expand_rules( /// the `Bindings` we should take. We push to the stack when we enter a /// repetition. /// -/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to +/// In other words, `Bindings` is a *multi* mapping from `Symbol` to /// `tt::TokenTree`, where the index to select a particular `TokenTree` among /// many is not a plain `usize`, but a `&[usize]`. #[derive(Debug, Default, Clone, PartialEq, Eq)] struct Bindings { - inner: FxHashMap<SmolStr, Binding>, + inner: FxHashMap<Symbol, Binding>, } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs index b20d5579ca6..3762d20bab6 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs @@ -61,9 +61,9 @@ use std::{rc::Rc, sync::Arc}; +use intern::{sym, Symbol}; use smallvec::{smallvec, SmallVec}; use span::{Edition, Span}; -use syntax::SmolStr; use tt::{iter::TtIter, DelimSpan}; use crate::{ @@ -74,12 +74,12 @@ use crate::{ }; impl Bindings { - fn push_optional(&mut self, name: &SmolStr) { - self.inner.insert(name.clone(), Binding::Fragment(Fragment::Empty)); + fn push_optional(&mut self, name: Symbol) { + self.inner.insert(name, Binding::Fragment(Fragment::Empty)); } - fn push_empty(&mut self, name: &SmolStr) { - self.inner.insert(name.clone(), Binding::Empty); + fn push_empty(&mut self, name: Symbol) { + self.inner.insert(name, Binding::Empty); } fn bindings(&self) -> impl Iterator<Item = &Binding> { @@ -127,10 +127,10 @@ pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree<Span>, edition: #[derive(Debug, Clone)] enum BindingKind { - Empty(SmolStr), - Optional(SmolStr), - Fragment(SmolStr, Fragment), - Missing(SmolStr, MetaVarKind), + Empty(Symbol), + Optional(Symbol), + Fragment(Symbol, Fragment), + Missing(Symbol, MetaVarKind), Nested(usize, usize), } @@ -178,20 +178,20 @@ impl BindingsBuilder { } } - fn push_empty(&mut self, idx: &mut BindingsIdx, var: &SmolStr) { + fn push_empty(&mut self, idx: &mut BindingsIdx, var: &Symbol) { self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Empty(var.clone())))); } - fn push_optional(&mut self, idx: &mut BindingsIdx, var: &SmolStr) { + fn push_optional(&mut self, idx: &mut BindingsIdx, var: &Symbol) { self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Optional(var.clone())))); } - fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment) { + fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &Symbol, fragment: Fragment) { self.nodes[idx.0] .push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment)))); } - fn push_missing(&mut self, idx: &mut BindingsIdx, var: &SmolStr, kind: MetaVarKind) { + fn push_missing(&mut self, idx: &mut BindingsIdx, var: &Symbol, kind: MetaVarKind) { self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Missing(var.clone(), kind)))); } @@ -219,10 +219,10 @@ impl BindingsBuilder { for cmd in nodes { match cmd { BindingKind::Empty(name) => { - bindings.push_empty(name); + bindings.push_empty(name.clone()); } BindingKind::Optional(name) => { - bindings.push_optional(name); + bindings.push_optional(name.clone()); } BindingKind::Fragment(name, fragment) => { bindings.inner.insert(name.clone(), Binding::Fragment(fragment.clone())); @@ -507,7 +507,7 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Literal(lhs)) => { if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(rhs, tt::Leaf::Literal(it) if it.text == lhs.text) { + if matches!(rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) { item.dot.next(); } else { res.add_err(ExpandError::UnexpectedToken); @@ -521,7 +521,7 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Ident(lhs)) => { if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(rhs, tt::Leaf::Ident(it) if it.text == lhs.text) { + if matches!(rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) { item.dot.next(); } else { res.add_err(ExpandError::UnexpectedToken); @@ -554,7 +554,7 @@ fn match_loop_inner<'t>( // ident, not a punct. ExpandError::UnexpectedToken } else { - let lhs: SmolStr = lhs.collect(); + let lhs = lhs.collect::<String>(); ExpandError::binding_error(format!("expected punct: `{lhs}`")) } } else { @@ -759,7 +759,9 @@ fn match_meta_var( // [1]: https://github.com/rust-lang/rust/blob/f0c4da499/compiler/rustc_expand/src/mbe/macro_parser.rs#L576 match input.peek_n(0) { Some(tt::TokenTree::Leaf(tt::Leaf::Ident(it))) - if it.text == "_" || it.text == "let" || it.text == "const" => + if it.sym == sym::underscore + || it.sym == sym::let_ + || it.sym == sym::const_ => { return ExpandResult::only_err(ExpandError::NoMatchingRule) } @@ -824,7 +826,7 @@ fn match_meta_var( expect_fragment(input, fragment, edition).map(|it| it.map(Fragment::Tokens)) } -fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) { +fn collect_vars(collector_fun: &mut impl FnMut(Symbol), pattern: &MetaTemplate) { for op in pattern.iter() { match op { Op::Var { name, .. } => collector_fun(name.clone()), @@ -908,13 +910,13 @@ fn expect_separator<S: Copy>(iter: &mut TtIter<'_, S>, separator: &Separator) -> let mut fork = iter.clone(); let ok = match separator { Separator::Ident(lhs) => match fork.expect_ident_or_underscore() { - Ok(rhs) => rhs.text == lhs.text, + Ok(rhs) => rhs.sym == lhs.sym, Err(_) => false, }, Separator::Literal(lhs) => match fork.expect_literal() { Ok(rhs) => match rhs { - tt::Leaf::Literal(rhs) => rhs.text == lhs.text, - tt::Leaf::Ident(rhs) => rhs.text == lhs.text, + tt::Leaf::Literal(rhs) => rhs.symbol == lhs.symbol, + tt::Leaf::Ident(rhs) => rhs.sym == lhs.symbol, tt::Leaf::Punct(_) => false, }, Err(_) => false, diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs index e3359865cb2..4de7cccd968 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs @@ -1,8 +1,8 @@ //! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` +use intern::{sym, Symbol}; use span::Span; -use syntax::{format_smolstr, SmolStr}; use tt::Delimiter; use crate::{ @@ -12,16 +12,16 @@ use crate::{ }; impl Bindings { - fn get(&self, name: &str) -> Result<&Binding, ExpandError> { + fn get(&self, name: &Symbol) -> Result<&Binding, ExpandError> { match self.inner.get(name) { Some(binding) => Ok(binding), - None => Err(ExpandError::UnresolvedBinding(Box::new(Box::from(name)))), + None => Err(ExpandError::UnresolvedBinding(Box::new(Box::from(name.as_str())))), } } fn get_fragment( &self, - name: &str, + name: &Symbol, mut span: Span, nesting: &mut [NestingState], marker: impl Fn(&mut Span), @@ -97,7 +97,7 @@ impl Bindings { | MetaVarKind::Expr | MetaVarKind::Ident => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_static("missing"), + sym: sym::missing.clone(), span, is_raw: tt::IdentIsRaw::No, }))) @@ -112,7 +112,7 @@ impl Bindings { spacing: tt::Spacing::Joint, })), tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_static("missing"), + sym: sym::missing.clone(), span, is_raw: tt::IdentIsRaw::No, })), @@ -121,7 +121,7 @@ impl Bindings { } MetaVarKind::Literal => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_static("\"missing\""), + sym: sym::missing.clone(), span, is_raw: tt::IdentIsRaw::No, }))) @@ -239,7 +239,7 @@ fn expand_subtree( ctx.nesting.get(ctx.nesting.len() - 1 - depth).map_or(0, |nest| nest.idx); arena.push( tt::Leaf::Literal(tt::Literal { - text: format_smolstr!("{index}"), + symbol: Symbol::integer(index), span: ctx.call_site, kind: tt::LitKind::Integer, suffix: None, @@ -254,7 +254,7 @@ fn expand_subtree( }); arena.push( tt::Leaf::Literal(tt::Literal { - text: format_smolstr!("{length}"), + symbol: Symbol::integer(length), span: ctx.call_site, kind: tt::LitKind::Integer, suffix: None, @@ -263,7 +263,7 @@ fn expand_subtree( ); } Op::Count { name, depth } => { - let mut binding = match ctx.bindings.get(name.as_str()) { + let mut binding = match ctx.bindings.get(name) { Ok(b) => b, Err(e) => { if err.is_none() { @@ -321,7 +321,7 @@ fn expand_subtree( }; arena.push( tt::Leaf::Literal(tt::Literal { - text: format_smolstr!("{c}"), + symbol: Symbol::integer(c), span: ctx.call_site, suffix: None, kind: tt::LitKind::Integer, @@ -344,12 +344,12 @@ fn expand_subtree( fn expand_var( ctx: &mut ExpandCtx<'_>, - v: &SmolStr, + v: &Symbol, id: Span, marker: impl Fn(&mut Span), ) -> ExpandResult<Fragment> { // We already handle $crate case in mbe parser - debug_assert!(v != "crate"); + debug_assert!(*v != sym::crate_); match ctx.bindings.get_fragment(v, id, &mut ctx.nesting, marker) { Ok(it) => ExpandResult::ok(it), @@ -373,7 +373,7 @@ fn expand_var( tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) .into(), tt::Leaf::from(tt::Ident { - text: v.clone(), + sym: v.clone(), span: id, is_raw: tt::IdentIsRaw::No, }) diff --git a/src/tools/rust-analyzer/crates/mbe/src/parser.rs b/src/tools/rust-analyzer/crates/mbe/src/parser.rs index 18af35c1e29..74a2c771e20 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/parser.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/parser.rs @@ -4,8 +4,8 @@ use std::sync::Arc; use arrayvec::ArrayVec; +use intern::{sym, Symbol}; use span::{Edition, Span, SyntaxContextId}; -use syntax::SmolStr; use tt::iter::TtIter; use crate::ParseError; @@ -67,12 +67,12 @@ impl MetaTemplate { #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Op { Var { - name: SmolStr, + name: Symbol, kind: Option<MetaVarKind>, id: Span, }, Ignore { - name: SmolStr, + name: Symbol, id: Span, }, Index { @@ -82,7 +82,7 @@ pub(crate) enum Op { depth: usize, }, Count { - name: SmolStr, + name: Symbol, // FIXME: `usize`` once we drop support for 1.76 depth: Option<usize>, }, @@ -138,8 +138,8 @@ impl PartialEq for Separator { use Separator::*; match (self, other) { - (Ident(a), Ident(b)) => a.text == b.text, - (Literal(a), Literal(b)) => a.text == b.text, + (Ident(a), Ident(b)) => a.sym == b.sym, + (Literal(a), Literal(b)) => a.symbol == b.symbol, (Puncts(a), Puncts(b)) if a.len() == b.len() => { let a_iter = a.iter().map(|a| a.char); let b_iter = b.iter().map(|b| b.char); @@ -203,23 +203,23 @@ fn next_op( } }, tt::TokenTree::Leaf(leaf) => match leaf { - tt::Leaf::Ident(ident) if ident.text == "crate" => { + tt::Leaf::Ident(ident) if ident.sym == sym::crate_ => { // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. Op::Ident(tt::Ident { - text: "$crate".into(), + sym: sym::dollar_crate.clone(), span: ident.span, is_raw: tt::IdentIsRaw::No, }) } tt::Leaf::Ident(ident) => { let kind = eat_fragment_kind(edition, src, mode)?; - let name = ident.text.clone(); + let name = ident.sym.clone(); let id = ident.span; Op::Var { name, kind, id } } tt::Leaf::Literal(lit) if is_boolean_literal(lit) => { let kind = eat_fragment_kind(edition, src, mode)?; - let name = lit.text.clone(); + let name = lit.symbol.clone(); let id = lit.span; Op::Var { name, kind, id } } @@ -277,7 +277,7 @@ fn eat_fragment_kind( let ident = src .expect_ident() .map_err(|()| ParseError::unexpected("missing fragment specifier"))?; - let kind = match ident.text.as_str() { + let kind = match ident.sym.as_str() { "path" => MetaVarKind::Path, "ty" => MetaVarKind::Ty, "pat" => match edition(ident.span.ctx) { @@ -303,7 +303,7 @@ fn eat_fragment_kind( } fn is_boolean_literal(lit: &tt::Literal<Span>) -> bool { - matches!(lit.text.as_str(), "true" | "false") + matches!(lit.symbol.as_str(), "true" | "false") } fn parse_repeat(src: &mut TtIter<'_, Span>) -> Result<(Option<Separator>, RepeatKind), ParseError> { @@ -353,23 +353,23 @@ fn parse_metavar_expr(new_meta_vars: bool, src: &mut TtIter<'_, Span>) -> Result let mut args = TtIter::new(args); - let op = match &*func.text { - "ignore" => { + let op = match &func.sym { + s if sym::ignore == *s => { if new_meta_vars { args.expect_dollar()?; } let ident = args.expect_ident()?; - Op::Ignore { name: ident.text.clone(), id: ident.span } + Op::Ignore { name: ident.sym.clone(), id: ident.span } } - "index" => Op::Index { depth: parse_depth(&mut args)? }, - "len" => Op::Len { depth: parse_depth(&mut args)? }, - "count" => { + s if sym::index == *s => Op::Index { depth: parse_depth(&mut args)? }, + s if sym::len == *s => Op::Len { depth: parse_depth(&mut args)? }, + s if sym::count == *s => { if new_meta_vars { args.expect_dollar()?; } let ident = args.expect_ident()?; let depth = if try_eat_comma(&mut args) { Some(parse_depth(&mut args)?) } else { None }; - Op::Count { name: ident.text.clone(), depth } + Op::Count { name: ident.sym.clone(), depth } } _ => return Err(()), }; @@ -384,11 +384,11 @@ fn parse_metavar_expr(new_meta_vars: bool, src: &mut TtIter<'_, Span>) -> Result fn parse_depth(src: &mut TtIter<'_, Span>) -> Result<usize, ()> { if src.len() == 0 { Ok(0) - } else if let tt::Leaf::Literal(tt::Literal { text, suffix: None, .. }) = + } else if let tt::Leaf::Literal(tt::Literal { symbol: text, suffix: None, .. }) = src.expect_literal()? { // Suffixes are not allowed. - text.parse().map_err(|_| ()) + text.as_str().parse().map_err(|_| ()) } else { Err(()) } diff --git a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs index 4d66464932b..7ff89631080 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs @@ -2,6 +2,7 @@ use std::fmt; +use intern::Symbol; use rustc_hash::{FxHashMap, FxHashSet}; use span::{Edition, SpanAnchor, SpanData, SpanMap}; use stdx::{format_to, never, non_empty_vec::NonEmptyVec}; @@ -322,7 +323,7 @@ where () => { tt::Ident { span: conv.span_for(abs_range), - text: token.to_text(conv), + sym: Symbol::intern(&token.to_text(conv)), is_raw: tt::IdentIsRaw::No, } .into() @@ -332,14 +333,14 @@ where T![true] | T![false] => make_ident!(), IDENT => { let text = token.to_text(conv); - tt::Ident::new(text, conv.span_for(abs_range)).into() + tt::Ident::new(&text, conv.span_for(abs_range)).into() } UNDERSCORE => make_ident!(), k if k.is_keyword() => make_ident!(), k if k.is_literal() => { let text = token.to_text(conv); let span = conv.span_for(abs_range); - token_to_literal(text, span).into() + token_to_literal(&text, span).into() } LIFETIME_IDENT => { let apostrophe = tt::Leaf::from(tt::Punct { @@ -351,7 +352,7 @@ where token_trees.push(apostrophe.into()); let ident = tt::Leaf::from(tt::Ident { - text: SmolStr::new(&token.to_text(conv)[1..]), + sym: Symbol::intern(&token.to_text(conv)[1..]), span: conv.span_for(TextRange::new( abs_range.start() + TextSize::of('\''), abs_range.end(), @@ -436,7 +437,7 @@ fn is_single_token_op(kind: SyntaxKind) -> bool { /// And then quote the string, which is needed to convert to `tt::Literal` /// /// Note that proc-macros desugar with string literals where as macro_rules macros desugar with raw string literals. -pub fn desugar_doc_comment_text(text: &str, mode: DocCommentDesugarMode) -> (SmolStr, tt::LitKind) { +pub fn desugar_doc_comment_text(text: &str, mode: DocCommentDesugarMode) -> (Symbol, tt::LitKind) { match mode { DocCommentDesugarMode::Mbe => { let mut num_of_hashes = 0; @@ -451,11 +452,11 @@ pub fn desugar_doc_comment_text(text: &str, mode: DocCommentDesugarMode) -> (Smo } // Quote raw string with delimiters - (text.into(), tt::LitKind::StrRaw(num_of_hashes)) + (Symbol::intern(text), tt::LitKind::StrRaw(num_of_hashes)) } // Quote string with delimiters DocCommentDesugarMode::ProcMacro => { - (format_smolstr!("{}", text.escape_debug()), tt::LitKind::Str) + (Symbol::intern(&format_smolstr!("{}", text.escape_debug())), tt::LitKind::Str) } } } @@ -471,7 +472,7 @@ fn convert_doc_comment<S: Copy>( let mk_ident = |s: &str| { tt::TokenTree::from(tt::Leaf::from(tt::Ident { - text: s.into(), + sym: Symbol::intern(s), span, is_raw: tt::IdentIsRaw::No, })) @@ -494,7 +495,7 @@ fn convert_doc_comment<S: Copy>( text = &text[0..text.len() - 2]; } let (text, kind) = desugar_doc_comment_text(text, mode); - let lit = tt::Literal { text, span, kind, suffix: None }; + let lit = tt::Literal { symbol: text, span, kind, suffix: None }; tt::TokenTree::from(tt::Leaf::from(lit)) }; @@ -928,7 +929,12 @@ where fn float_split(&mut self, has_pseudo_dot: bool) { let (text, span) = match self.cursor.token_tree() { Some(tt::buffer::TokenTreeRef::Leaf( - tt::Leaf::Literal(tt::Literal { text, span, kind: tt::LitKind::Float, suffix: _ }), + tt::Leaf::Literal(tt::Literal { + symbol: text, + span, + kind: tt::LitKind::Float, + suffix: _, + }), _, )) => (text.as_str(), *span), tt => unreachable!("{tt:?}"), @@ -988,7 +994,7 @@ where self.buf.push_str("r#"); self.text_pos += TextSize::of("r#"); } - let r = (ident.text.as_str(), ident.span); + let r = (ident.sym.as_str(), ident.span); self.cursor = self.cursor.bump(); r } diff --git a/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs b/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs index bf5494d3714..7636359805c 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs @@ -49,23 +49,22 @@ pub(crate) fn to_parser_input<S: Copy + fmt::Debug>(buffer: &TokenBuffer<'_, S>) }; res.push(kind); - if kind == FLOAT_NUMBER && !lit.text.ends_with('.') { + if kind == FLOAT_NUMBER && !lit.symbol.as_str().ends_with('.') { // Tag the token as joint if it is float with a fractional part // we use this jointness to inform the parser about what token split // event to emit when we encounter a float literal in a field access res.was_joint(); } } - tt::Leaf::Ident(ident) => match ident.text.as_ref() { + tt::Leaf::Ident(ident) => match ident.sym.as_str() { "_" => res.push(T![_]), i if i.starts_with('\'') => res.push(LIFETIME_IDENT), _ if ident.is_raw.yes() => res.push(IDENT), - _ => match SyntaxKind::from_keyword(&ident.text) { + text => match SyntaxKind::from_keyword(text) { Some(kind) => res.push(kind), None => { - let contextual_keyword = - SyntaxKind::from_contextual_keyword(&ident.text) - .unwrap_or(SyntaxKind::IDENT); + let contextual_keyword = SyntaxKind::from_contextual_keyword(text) + .unwrap_or(SyntaxKind::IDENT); res.push_ident(contextual_keyword); } }, diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/Cargo.toml b/src/tools/rust-analyzer/crates/proc-macro-api/Cargo.toml index 889eefa8b5c..345fb9f8ae9 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/Cargo.toml +++ b/src/tools/rust-analyzer/crates/proc-macro-api/Cargo.toml @@ -28,8 +28,7 @@ span.workspace = true # InternIds for the syntax context base-db.workspace = true la-arena.workspace = true -# only here to parse via token_to_literal -mbe.workspace = true +intern.workspace = true [lints] workspace = true diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs index 3a915e668bb..54c1475b8b1 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs @@ -13,7 +13,6 @@ use base_db::Env; use paths::{AbsPath, AbsPathBuf}; use span::Span; use std::{fmt, io, sync::Arc}; -use tt::SmolStr; use serde::{Deserialize, Serialize}; @@ -66,7 +65,7 @@ impl MacroDylib { pub struct ProcMacro { process: Arc<ProcMacroProcessSrv>, dylib_path: Arc<AbsPathBuf>, - name: SmolStr, + name: Box<str>, kind: ProcMacroKind, } diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/msg.rs index 65835048173..65f60a7c5bd 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/msg.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/msg.rs @@ -158,6 +158,7 @@ type ProtocolWrite<W: Write> = for<'o, 'msg> fn(out: &'o mut W, msg: &'msg str) #[cfg(test)] mod tests { use base_db::FileId; + use intern::{sym, Symbol}; use la_arena::RawIdx; use span::{ErasedFileAstId, Span, SpanAnchor, SyntaxContextId}; use text_size::{TextRange, TextSize}; @@ -174,7 +175,7 @@ mod tests { let token_trees = Box::new([ TokenTree::Leaf( Ident { - text: "struct".into(), + sym: Symbol::intern("struct"), span: Span { range: TextRange::at(TextSize::new(0), TextSize::of("struct")), anchor, @@ -186,7 +187,7 @@ mod tests { ), TokenTree::Leaf( Ident { - text: "Foo".into(), + sym: Symbol::intern("Foo"), span: Span { range: TextRange::at(TextSize::new(5), TextSize::of("r#Foo")), anchor, @@ -197,7 +198,7 @@ mod tests { .into(), ), TokenTree::Leaf(Leaf::Literal(Literal { - text: "Foo".into(), + symbol: Symbol::intern("Foo"), span: Span { range: TextRange::at(TextSize::new(10), TextSize::of("\"Foo\"")), anchor, @@ -230,14 +231,14 @@ mod tests { kind: DelimiterKind::Brace, }, token_trees: Box::new([TokenTree::Leaf(Leaf::Literal(Literal { - text: "0".into(), + symbol: sym::INTEGER_0.clone(), span: Span { range: TextRange::at(TextSize::new(15), TextSize::of("0u32")), anchor, ctx: SyntaxContextId::ROOT, }, kind: tt::LitKind::Integer, - suffix: Some(Box::new("u32".into())), + suffix: Some(sym::u32.clone()), }))]), }), ]); diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/msg/flat.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/msg/flat.rs index 3d962e99d92..19260bc817d 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/msg/flat.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/msg/flat.rs @@ -37,6 +37,7 @@ use std::collections::VecDeque; +use intern::Symbol; use la_arena::RawIdx; use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; @@ -433,8 +434,8 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> { let id = self.token_id_of(lit.span); let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { ( - self.intern(&lit.text), - lit.suffix.as_ref().map(|s| self.intern(s)).unwrap_or(!0), + self.intern(lit.symbol.as_str()), + lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), ) } else { (self.intern_owned(format!("{lit}")), !0) @@ -469,11 +470,11 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> { let idx = self.ident.len() as u32; let id = self.token_id_of(ident.span); let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern(&ident.text) + self.intern(ident.sym.as_str()) } else if ident.is_raw.yes() { - self.intern_owned(format!("r#{}", ident.text,)) + self.intern_owned(format!("r#{}", ident.sym.as_str(),)) } else { - self.intern(&ident.text) + self.intern(ident.sym.as_str()) }; self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); idx << 2 | 0b11 @@ -555,7 +556,7 @@ impl<'span, S: InternableSpan> Reader<'span, S> { let span = read_span(repr.id); tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { tt::Literal { - text: text.into(), + symbol: Symbol::intern(text), span, kind: match u16::to_le_bytes(repr.kind) { [0, _] => Err(()), @@ -572,15 +573,15 @@ impl<'span, S: InternableSpan> Reader<'span, S> { _ => unreachable!(), }, suffix: if repr.suffix != !0 { - Some(Box::new( - self.text[repr.suffix as usize].as_str().into(), + Some(Symbol::intern( + self.text[repr.suffix as usize].as_str(), )) } else { None }, } } else { - tt::token_to_literal(text.into(), span) + tt::token_to_literal(text, span) }) .into() } @@ -609,7 +610,7 @@ impl<'span, S: InternableSpan> Reader<'span, S> { tt::IdentIsRaw::split_from_symbol(text) }; tt::Leaf::Ident(tt::Ident { - text: text.into(), + sym: Symbol::intern(text), span: read_span(repr.id), is_raw, }) diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml b/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml index 065701c05cc..673b5bd78a8 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml @@ -25,6 +25,7 @@ base-db.workspace = true span.workspace = true proc-macro-api.workspace = true ra-ap-rustc_lexer.workspace = true +intern.workspace = true [dev-dependencies] expect-test = "1.4.0" diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs index e6281035e1a..f0aa6b3f93f 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs @@ -130,14 +130,13 @@ impl ProcMacroSrvSpan for TokenId { type Server = server_impl::token_id::TokenIdServer; fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server { - Self::Server { interner: &server_impl::SYMBOL_INTERNER, call_site, def_site, mixed_site } + Self::Server { call_site, def_site, mixed_site } } } impl ProcMacroSrvSpan for Span { type Server = server_impl::rust_analyzer_span::RaSpanServer; fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server { Self::Server { - interner: &server_impl::SYMBOL_INTERNER, call_site, def_site, mixed_site, diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs index 68e0f85978e..c9a86216905 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs @@ -14,9 +14,9 @@ mod token_stream; pub use token_stream::TokenStream; pub mod rust_analyzer_span; -mod symbol; +// mod symbol; pub mod token_id; -pub use symbol::*; +// pub use symbol::*; use tt::Spacing; fn delim_to_internal<S>(d: proc_macro::Delimiter, span: bridge::DelimSpan<S>) -> tt::Delimiter<S> { diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index 1a71f39612d..452ad8e083d 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -10,13 +10,14 @@ use std::{ ops::{Bound, Range}, }; +use intern::Symbol; use proc_macro::bridge::{self, server}; use span::{Span, FIXUP_ERASED_FILE_AST_ID_MARKER}; use tt::{TextRange, TextSize}; use crate::server_impl::{ delim_to_external, delim_to_internal, literal_kind_to_external, literal_kind_to_internal, - token_stream::TokenStreamBuilder, Symbol, SymbolInternerRef, SYMBOL_INTERNER, + token_stream::TokenStreamBuilder, }; mod tt { pub use tt::*; @@ -36,7 +37,6 @@ pub struct SourceFile; pub struct FreeFunctions; pub struct RaSpanServer { - pub(crate) interner: SymbolInternerRef, // FIXME: Report this back to the caller to track as dependencies pub tracked_env_vars: HashMap<Box<str>, Option<Box<str>>>, // FIXME: Report this back to the caller to track as dependencies @@ -126,15 +126,10 @@ impl server::FreeFunctions for RaSpanServer { let lit = &lit[start_offset..lit.len() - end_offset]; let suffix = match suffix { "" | "_" => None, - suffix => Some(Symbol::intern(self.interner, suffix)), + suffix => Some(Symbol::intern(suffix)), }; - Ok(bridge::Literal { - kind, - symbol: Symbol::intern(self.interner, lit), - suffix, - span: self.call_site, - }) + Ok(bridge::Literal { kind, symbol: Symbol::intern(lit), suffix, span: self.call_site }) } fn emit_diagnostic(&mut self, _: bridge::Diagnostic<Self::Span>) { @@ -170,9 +165,9 @@ impl server::TokenStream for RaSpanServer { } bridge::TokenTree::Ident(ident) => { - let text = ident.sym.text(self.interner); + let text = ident.sym; let ident: tt::Ident = tt::Ident { - text, + sym: text, span: ident.span, is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, }; @@ -183,8 +178,8 @@ impl server::TokenStream for RaSpanServer { bridge::TokenTree::Literal(literal) => { let literal = tt::Literal { - text: literal.symbol.text(self.interner), - suffix: literal.suffix.map(|it| Box::new(it.text(self.interner))), + symbol: literal.symbol, + suffix: literal.suffix, span: literal.span, kind: literal_kind_to_internal(literal.kind), }; @@ -255,7 +250,7 @@ impl server::TokenStream for RaSpanServer { .map(|tree| match tree { tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { bridge::TokenTree::Ident(bridge::Ident { - sym: Symbol::intern(self.interner, &ident.text), + sym: ident.sym, is_raw: ident.is_raw.yes(), span: ident.span, }) @@ -264,8 +259,8 @@ impl server::TokenStream for RaSpanServer { bridge::TokenTree::Literal(bridge::Literal { span: lit.span, kind: literal_kind_to_external(lit.kind), - symbol: Symbol::intern(self.interner, &lit.text), - suffix: lit.suffix.map(|it| Symbol::intern(self.interner, &it)), + symbol: lit.symbol, + suffix: lit.suffix, }) } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { @@ -464,12 +459,95 @@ impl server::Server for RaSpanServer { } fn intern_symbol(ident: &str) -> Self::Symbol { - // FIXME: should be `self.interner` once the proc-macro api allows it. - Symbol::intern(&SYMBOL_INTERNER, &::tt::SmolStr::from(ident)) + Symbol::intern(ident) } fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { - // FIXME: should be `self.interner` once the proc-macro api allows it. - f(symbol.text(&SYMBOL_INTERNER).as_str()) + f(symbol.as_str()) + } +} + +#[cfg(test)] +mod tests { + use span::SyntaxContextId; + + use super::*; + + #[test] + fn test_ra_server_to_string() { + let span = Span { + range: TextRange::empty(TextSize::new(0)), + anchor: span::SpanAnchor { + file_id: span::FileId::from_raw(0), + ast_id: span::ErasedFileAstId::from_raw(0.into()), + }, + ctx: SyntaxContextId::ROOT, + }; + let s = TokenStream { + token_trees: vec![ + tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + sym: Symbol::intern("struct"), + span, + is_raw: tt::IdentIsRaw::No, + })), + tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + sym: Symbol::intern("T"), + span: span, + is_raw: tt::IdentIsRaw::No, + })), + tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter { + open: span, + close: span, + kind: tt::DelimiterKind::Brace, + }, + token_trees: Box::new([]), + }), + ], + }; + + assert_eq!(s.to_string(), "struct T {}"); + } + + #[test] + fn test_ra_server_from_str() { + let span = Span { + range: TextRange::empty(TextSize::new(0)), + anchor: span::SpanAnchor { + file_id: span::FileId::from_raw(0), + ast_id: span::ErasedFileAstId::from_raw(0.into()), + }, + ctx: SyntaxContextId::ROOT, + }; + let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter { + open: span, + close: span, + kind: tt::DelimiterKind::Parenthesis, + }, + token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + is_raw: tt::IdentIsRaw::No, + sym: Symbol::intern("a"), + span, + }))]), + }); + + let t1 = TokenStream::from_str("(a)", span).unwrap(); + assert_eq!(t1.token_trees.len(), 1); + assert_eq!(t1.token_trees[0], subtree_paren_a); + + let t2 = TokenStream::from_str("(a);", span).unwrap(); + assert_eq!(t2.token_trees.len(), 2); + assert_eq!(t2.token_trees[0], subtree_paren_a); + + let underscore = TokenStream::from_str("_", span).unwrap(); + assert_eq!( + underscore.token_trees[0], + tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + sym: Symbol::intern("_"), + span, + is_raw: tt::IdentIsRaw::No, + })) + ); } } diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/symbol.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/symbol.rs index 540d06457f2..6863ce95997 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/symbol.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/symbol.rs @@ -1,7 +1,6 @@ //! Symbol interner for proc-macro-srv use std::{cell::RefCell, collections::HashMap, thread::LocalKey}; -use tt::SmolStr; thread_local! { pub(crate) static SYMBOL_INTERNER: RefCell<SymbolInterner> = Default::default(); diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs index 94d5748b087..7720c6d83c3 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs @@ -5,11 +5,12 @@ use std::{ ops::{Bound, Range}, }; +use intern::Symbol; use proc_macro::bridge::{self, server}; use crate::server_impl::{ delim_to_external, delim_to_internal, literal_kind_to_external, literal_kind_to_internal, - token_stream::TokenStreamBuilder, Symbol, SymbolInternerRef, SYMBOL_INTERNER, + token_stream::TokenStreamBuilder, }; mod tt { pub use proc_macro_api::msg::TokenId; @@ -36,7 +37,6 @@ pub struct SourceFile; pub struct FreeFunctions; pub struct TokenIdServer { - pub(crate) interner: SymbolInternerRef, pub call_site: Span, pub def_site: Span, pub mixed_site: Span, @@ -117,15 +117,10 @@ impl server::FreeFunctions for TokenIdServer { let lit = &lit[start_offset..lit.len() - end_offset]; let suffix = match suffix { "" | "_" => None, - suffix => Some(Symbol::intern(self.interner, suffix)), + suffix => Some(Symbol::intern(suffix)), }; - Ok(bridge::Literal { - kind, - symbol: Symbol::intern(self.interner, lit), - suffix, - span: self.call_site, - }) + Ok(bridge::Literal { kind, symbol: Symbol::intern(lit), suffix, span: self.call_site }) } fn emit_diagnostic(&mut self, _: bridge::Diagnostic<Self::Span>) {} @@ -159,9 +154,8 @@ impl server::TokenStream for TokenIdServer { } bridge::TokenTree::Ident(ident) => { - let text = ident.sym.text(self.interner); let ident: tt::Ident = tt::Ident { - text, + sym: ident.sym, span: ident.span, is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, }; @@ -172,8 +166,8 @@ impl server::TokenStream for TokenIdServer { bridge::TokenTree::Literal(literal) => { let literal = Literal { - text: literal.symbol.text(self.interner), - suffix: literal.suffix.map(|it| Box::new(it.text(self.interner))), + symbol: literal.symbol, + suffix: literal.suffix, span: literal.span, kind: literal_kind_to_internal(literal.kind), }; @@ -239,7 +233,7 @@ impl server::TokenStream for TokenIdServer { .map(|tree| match tree { tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { bridge::TokenTree::Ident(bridge::Ident { - sym: Symbol::intern(self.interner, &ident.text), + sym: ident.sym, is_raw: ident.is_raw.yes(), span: ident.span, }) @@ -248,8 +242,8 @@ impl server::TokenStream for TokenIdServer { bridge::TokenTree::Literal(bridge::Literal { span: lit.span, kind: literal_kind_to_external(lit.kind), - symbol: Symbol::intern(self.interner, &lit.text), - suffix: lit.suffix.map(|it| Symbol::intern(self.interner, &it)), + symbol: lit.symbol, + suffix: lit.suffix, }) } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { @@ -366,11 +360,11 @@ impl server::Server for TokenIdServer { } fn intern_symbol(ident: &str) -> Self::Symbol { - Symbol::intern(&SYMBOL_INTERNER, &::tt::SmolStr::from(ident)) + Symbol::intern(ident) } fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { - f(symbol.text(&SYMBOL_INTERNER).as_str()) + f(symbol.as_str()) } } @@ -383,12 +377,12 @@ mod tests { let s = TokenStream { token_trees: vec![ tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: "struct".into(), + sym: Symbol::intern("struct"), span: tt::TokenId(0), is_raw: tt::IdentIsRaw::No, })), tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: "T".into(), + sym: Symbol::intern("T"), span: tt::TokenId(0), is_raw: tt::IdentIsRaw::No, })), @@ -416,7 +410,7 @@ mod tests { }, token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { is_raw: tt::IdentIsRaw::No, - text: "a".into(), + sym: Symbol::intern("a"), span: tt::TokenId(0), }))]), }); @@ -433,7 +427,7 @@ mod tests { assert_eq!( underscore.token_trees[0], tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: "_".into(), + sym: Symbol::intern("_"), span: tt::TokenId(0), is_raw: tt::IdentIsRaw::No, })) diff --git a/src/tools/rust-analyzer/crates/test-fixture/Cargo.toml b/src/tools/rust-analyzer/crates/test-fixture/Cargo.toml index 35e39229894..f9565721dd5 100644 --- a/src/tools/rust-analyzer/crates/test-fixture/Cargo.toml +++ b/src/tools/rust-analyzer/crates/test-fixture/Cargo.toml @@ -16,6 +16,7 @@ base-db.workspace = true rustc-hash.workspace = true span.workspace = true stdx.workspace = true +intern.workspace = true [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs index e65186d3771..7a0c474b750 100644 --- a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs +++ b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs @@ -13,6 +13,7 @@ use hir_expand::{ ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacroKind, ProcMacros, }, }; +use intern::Symbol; use rustc_hash::FxHashMap; use span::{Edition, FileId, FilePosition, FileRange, Span}; use test_utils::{ @@ -640,11 +641,11 @@ impl ProcMacroExpander for ShortenProcMacroExpander { Leaf::Literal(it) => { // XXX Currently replaces any literals with an empty string, but supporting // "shortening" other literals would be nice. - it.text = "\"\"".into(); + it.symbol = Symbol::empty(); } Leaf::Punct(_) => {} Leaf::Ident(it) => { - it.text = it.text.chars().take(1).collect(); + it.sym = Symbol::intern(&it.sym.as_str().chars().take(1).collect::<String>()); } } leaf diff --git a/src/tools/rust-analyzer/crates/tt/Cargo.toml b/src/tools/rust-analyzer/crates/tt/Cargo.toml index 1900635b995..cea1519c2dd 100644 --- a/src/tools/rust-analyzer/crates/tt/Cargo.toml +++ b/src/tools/rust-analyzer/crates/tt/Cargo.toml @@ -13,10 +13,10 @@ doctest = false [dependencies] arrayvec.workspace = true -smol_str.workspace = true text-size.workspace = true stdx.workspace = true +intern.workspace = true ra-ap-rustc_lexer.workspace = true [features] diff --git a/src/tools/rust-analyzer/crates/tt/src/iter.rs b/src/tools/rust-analyzer/crates/tt/src/iter.rs index 175259a3e47..c0195b835ac 100644 --- a/src/tools/rust-analyzer/crates/tt/src/iter.rs +++ b/src/tools/rust-analyzer/crates/tt/src/iter.rs @@ -2,6 +2,7 @@ //! macro definition into a list of patterns and templates. use arrayvec::ArrayVec; +use intern::sym; use crate::{Ident, Leaf, Punct, Spacing, Subtree, TokenTree}; @@ -58,7 +59,7 @@ impl<'a, S: Copy> TtIter<'a, S> { pub fn expect_ident(&mut self) -> Result<&'a Ident<S>, ()> { match self.expect_leaf()? { - Leaf::Ident(it) if it.text != "_" => Ok(it), + Leaf::Ident(it) if it.sym != sym::underscore => Ok(it), _ => Err(()), } } @@ -74,7 +75,7 @@ impl<'a, S: Copy> TtIter<'a, S> { let it = self.expect_leaf()?; match it { Leaf::Literal(_) => Ok(it), - Leaf::Ident(ident) if ident.text == "true" || ident.text == "false" => Ok(it), + Leaf::Ident(ident) if ident.sym == sym::true_ || ident.sym == sym::false_ => Ok(it), _ => Err(()), } } diff --git a/src/tools/rust-analyzer/crates/tt/src/lib.rs b/src/tools/rust-analyzer/crates/tt/src/lib.rs index c328b3f8a3c..7b72f9ff108 100644 --- a/src/tools/rust-analyzer/crates/tt/src/lib.rs +++ b/src/tools/rust-analyzer/crates/tt/src/lib.rs @@ -14,16 +14,16 @@ pub mod iter; use std::fmt; +use intern::Symbol; use stdx::{impl_from, itertools::Itertools as _}; -pub use smol_str::{format_smolstr, SmolStr}; pub use text_size::{TextRange, TextSize}; #[derive(Clone, PartialEq, Debug)] pub struct Lit { pub kind: LitKind, - pub symbol: SmolStr, - pub suffix: Option<SmolStr>, + pub symbol: Symbol, + pub suffix: Option<Symbol>, } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] @@ -35,6 +35,9 @@ impl IdentIsRaw { pub fn yes(self) -> bool { matches!(self, IdentIsRaw::Yes) } + pub fn no(&self) -> bool { + matches!(self, IdentIsRaw::No) + } pub fn as_str(self) -> &'static str { match self { IdentIsRaw::No => "", @@ -197,25 +200,30 @@ pub enum DelimiterKind { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Literal<S> { // escaped - pub text: SmolStr, + pub symbol: Symbol, pub span: S, pub kind: LitKind, - pub suffix: Option<Box<SmolStr>>, + pub suffix: Option<Symbol>, } -pub fn token_to_literal<S>(text: SmolStr, span: S) -> Literal<S> +pub fn token_to_literal<S>(text: &str, span: S) -> Literal<S> where S: Copy, { use rustc_lexer::LiteralKind; - let token = rustc_lexer::tokenize(&text).next_tuple(); + let token = rustc_lexer::tokenize(text).next_tuple(); let Some((rustc_lexer::Token { kind: rustc_lexer::TokenKind::Literal { kind, suffix_start }, .. },)) = token else { - return Literal { span, text, kind: LitKind::Err(()), suffix: None }; + return Literal { + span, + symbol: Symbol::intern(text), + kind: LitKind::Err(()), + suffix: None, + }; }; let (kind, start_offset, end_offset) = match kind { @@ -247,10 +255,10 @@ where let lit = &lit[start_offset..lit.len() - end_offset]; let suffix = match suffix { "" | "_" => None, - suffix => Some(Box::new(suffix.into())), + suffix => Some(Symbol::intern(suffix)), }; - Literal { span, text: lit.into(), kind, suffix } + Literal { span, symbol: Symbol::intern(lit), kind, suffix } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -323,22 +331,16 @@ pub enum Spacing { /// Identifier or keyword. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Ident<S> { - pub text: SmolStr, + pub sym: Symbol, pub span: S, pub is_raw: IdentIsRaw, } impl<S> Ident<S> { - pub fn new(text: impl Into<SmolStr> + AsRef<str>, span: S) -> Self { - let t = text.as_ref(); + pub fn new(text: &str, span: S) -> Self { // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref()); - let raw_stripped = t.strip_prefix("r#"); - let is_raw = if raw_stripped.is_none() { IdentIsRaw::No } else { IdentIsRaw::Yes }; - let text = match raw_stripped { - Some(derawed) => derawed.into(), - None => text.into(), - }; - Ident { text, span, is_raw } + let (is_raw, text) = IdentIsRaw::split_from_symbol(text); + Ident { sym: Symbol::intern(text), span, is_raw } } } @@ -389,8 +391,8 @@ fn print_debug_token<S: fmt::Debug>( "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, - lit.text, - lit.suffix.as_ref().map(|it| &***it).unwrap_or(""), + lit.symbol, + lit.suffix.as_ref().map(|it| it.as_str()).unwrap_or(""), lit.span )?; } @@ -410,7 +412,7 @@ fn print_debug_token<S: fmt::Debug>( "{}IDENT {}{} {:#?}", align, ident.is_raw.as_str(), - ident.text, + ident.sym, ident.span )?; } @@ -479,26 +481,26 @@ impl<S> fmt::Display for Leaf<S> { impl<S> fmt::Display for Ident<S> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.is_raw.as_str(), f)?; - fmt::Display::fmt(&self.text, f) + fmt::Display::fmt(&self.sym, f) } } impl<S> fmt::Display for Literal<S> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.kind { - LitKind::Byte => write!(f, "b'{}'", self.text), - LitKind::Char => write!(f, "'{}'", self.text), - LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", self.text), - LitKind::Str => write!(f, "\"{}\"", self.text), - LitKind::ByteStr => write!(f, "b\"{}\"", self.text), - LitKind::CStr => write!(f, "c\"{}\"", self.text), + LitKind::Byte => write!(f, "b'{}'", self.symbol), + LitKind::Char => write!(f, "'{}'", self.symbol), + LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", self.symbol), + LitKind::Str => write!(f, "\"{}\"", self.symbol), + LitKind::ByteStr => write!(f, "b\"{}\"", self.symbol), + LitKind::CStr => write!(f, "c\"{}\"", self.symbol), LitKind::StrRaw(num_of_hashes) => { let num_of_hashes = num_of_hashes as usize; write!( f, r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", - text = self.text + text = self.symbol ) } LitKind::ByteStrRaw(num_of_hashes) => { @@ -507,7 +509,7 @@ impl<S> fmt::Display for Literal<S> { f, r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", - text = self.text + text = self.symbol ) } LitKind::CStrRaw(num_of_hashes) => { @@ -516,7 +518,7 @@ impl<S> fmt::Display for Literal<S> { f, r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", - text = self.text + text = self.symbol ) } }?; @@ -566,9 +568,9 @@ impl<S> Subtree<S> { let s = match child { TokenTree::Leaf(it) => { let s = match it { - Leaf::Literal(it) => it.text.to_string(), + Leaf::Literal(it) => it.symbol.to_string(), Leaf::Punct(it) => it.char.to_string(), - Leaf::Ident(it) => format!("{}{}", it.is_raw.as_str(), it.text), + Leaf::Ident(it) => format!("{}{}", it.is_raw.as_str(), it.sym), }; match (it, last) { (Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => { @@ -599,9 +601,9 @@ pub fn pretty<S>(tkns: &[TokenTree<S>]) -> String { fn tokentree_to_text<S>(tkn: &TokenTree<S>) -> String { match tkn { TokenTree::Leaf(Leaf::Ident(ident)) => { - format!("{}{}", ident.is_raw.as_str(), ident.text) + format!("{}{}", ident.is_raw.as_str(), ident.sym) } - TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(), + TokenTree::Leaf(Leaf::Literal(literal)) => literal.symbol.as_str().to_owned(), TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char), TokenTree::Subtree(subtree) => { let content = pretty(&subtree.token_trees); |