diff options
| author | Jeffrey Seyfried <jeffrey.seyfried@gmail.com> | 2016-11-16 08:21:52 +0000 |
|---|---|---|
| committer | Jeffrey Seyfried <jeffrey.seyfried@gmail.com> | 2016-11-20 23:40:20 +0000 |
| commit | d2f8fb0a0a9dd98ea9d6a01620f1a21f425236c0 (patch) | |
| tree | 941a81af819b4ff81b47fd0f50241477e9090d2d /src/libsyntax | |
| parent | f177a00ac92b871add90ca559a8591c4647a3c7e (diff) | |
| download | rust-d2f8fb0a0a9dd98ea9d6a01620f1a21f425236c0.tar.gz rust-d2f8fb0a0a9dd98ea9d6a01620f1a21f425236c0.zip | |
Move `syntax::util::interner` -> `syntax::symbol`, cleanup.
Diffstat (limited to 'src/libsyntax')
26 files changed, 487 insertions, 548 deletions
diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 45970a63ab4..a001985ded9 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -14,15 +14,16 @@ pub use self::TyParamBound::*; pub use self::UnsafeSource::*; pub use self::ViewPath_::*; pub use self::PathParameters::*; +pub use symbol::Symbol as Name; pub use util::ThinVec; use syntax_pos::{mk_sp, Span, DUMMY_SP, ExpnId}; use codemap::{respan, Spanned}; use abi::Abi; use ext::hygiene::SyntaxContext; -use parse::token::{self, keywords, InternedString}; use print::pprust; use ptr::P; +use symbol::{Symbol, keywords, InternedString}; use tokenstream::{TokenTree}; use std::collections::HashSet; @@ -32,60 +33,24 @@ use std::u32; use serialize::{self, Encodable, Decodable, Encoder, Decoder}; -/// A name is a part of an identifier, representing a string or gensym. It's -/// the result of interning. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Name(pub u32); - /// An identifier contains a Name (index into the interner /// table) and a SyntaxContext to track renaming and /// macro expansion per Flatt et al., "Macros That Work Together" #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct Ident { - pub name: Name, + pub name: Symbol, pub ctxt: SyntaxContext } -impl Name { - pub fn as_str(self) -> token::InternedString { - token::InternedString::new_from_name(self) - } -} - -impl fmt::Debug for Name { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}({})", self, self.0) - } -} - -impl fmt::Display for Name { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.as_str(), f) - } -} - -impl Encodable for Name { - fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_str(&self.as_str()) - } -} - -impl Decodable for Name { - fn decode<D: Decoder>(d: &mut D) -> Result<Name, D::Error> { - Ok(token::intern(&d.read_str()?)) - } -} - -impl<'a> ::std::cmp::PartialEq<&'a str> for Name { - fn eq(&self, other: &&str) -> bool { - *self.as_str() == **other - } -} - impl Ident { pub const fn with_empty_ctxt(name: Name) -> Ident { Ident { name: name, ctxt: SyntaxContext::empty() } } + + /// Maps a string to an identifier with an empty syntax context. + pub fn from_str(s: &str) -> Ident { + Ident::with_empty_ctxt(Symbol::intern(s)) + } } impl fmt::Debug for Ident { diff --git a/src/libsyntax/attr.rs b/src/libsyntax/attr.rs index cd93b8e96e4..4fce739efe2 100644 --- a/src/libsyntax/attr.rs +++ b/src/libsyntax/attr.rs @@ -23,9 +23,9 @@ use syntax_pos::{Span, BytePos, DUMMY_SP}; use errors::Handler; use feature_gate::{Features, GatedCfg}; use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use parse::token::InternedString; -use parse::{ParseSess, token}; +use parse::ParseSess; use ptr::P; +use symbol::{self, Symbol, InternedString}; use util::ThinVec; use std::cell::{RefCell, Cell}; @@ -278,8 +278,8 @@ impl Attribute { if self.is_sugared_doc { let comment = self.value_str().unwrap(); let meta = mk_name_value_item_str( - token::intern("doc"), - token::intern_and_get_ident(&strip_doc_comment_decoration( + Symbol::intern("doc"), + symbol::intern_and_get_ident(&strip_doc_comment_decoration( &comment))); if self.style == ast::AttrStyle::Outer { f(&mk_attr_outer(self.id, meta)) @@ -392,7 +392,7 @@ pub fn mk_sugared_doc_attr(id: AttrId, text: InternedString, lo: BytePos, hi: By style: style, value: MetaItem { span: mk_sp(lo, hi), - name: token::intern("doc"), + name: Symbol::intern("doc"), node: MetaItemKind::NameValue(lit), }, is_sugared_doc: true, diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 81c8e0bdb82..aa81d7afcb3 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -19,6 +19,7 @@ use ext::base::{ExtCtxt, MacEager, MacResult}; use ext::build::AstBuilder; use parse::token; use ptr::P; +use symbol::Symbol; use tokenstream::{TokenTree}; use util::small_vector::SmallVector; @@ -141,7 +142,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt, )); } }); - let sym = Ident::with_empty_ctxt(token::gensym(&format!( + let sym = Ident::with_empty_ctxt(Symbol::gensym(&format!( "__register_diagnostic_{}", code ))); MacEager::items(SmallVector::many(vec![ diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 7f66b060052..0fd360ba2e4 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -20,8 +20,8 @@ use ext::hygiene::Mark; use fold::{self, Folder}; use parse::{self, parser}; use parse::token; -use parse::token::{InternedString, str_to_ident}; use ptr::P; +use symbol::{Symbol, InternedString}; use util::small_vector::SmallVector; use std::path::PathBuf; @@ -735,7 +735,7 @@ impl<'a> ExtCtxt<'a> { self.ecfg.trace_mac = x } pub fn ident_of(&self, st: &str) -> ast::Ident { - str_to_ident(st) + ast::Ident::from_str(st) } pub fn std_path(&self, components: &[&str]) -> Vec<ast::Ident> { let mut v = Vec::new(); @@ -746,7 +746,7 @@ impl<'a> ExtCtxt<'a> { return v } pub fn name_of(&self, st: &str) -> ast::Name { - token::intern(st) + Symbol::intern(st) } } diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index f488e1cf95c..b96a4624508 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -14,8 +14,8 @@ use attr; use syntax_pos::{Span, DUMMY_SP, Pos}; use codemap::{dummy_spanned, respan, Spanned}; use ext::base::ExtCtxt; -use parse::token::{self, keywords, InternedString}; use ptr::P; +use symbol::{intern_and_get_ident, keywords, InternedString}; // Transitional reexports so qquote can find the paths it is looking for mod syntax { @@ -787,8 +787,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn expr_fail(&self, span: Span, msg: InternedString) -> P<ast::Expr> { let loc = self.codemap().lookup_char_pos(span.lo); - let expr_file = self.expr_str(span, - token::intern_and_get_ident(&loc.file.name)); + let expr_file = self.expr_str(span, intern_and_get_ident(&loc.file.name)); let expr_line = self.expr_u32(span, loc.line as u32); let expr_file_line_tuple = self.expr_tuple(span, vec![expr_file, expr_line]); let expr_file_line_ptr = self.expr_addr_of(span, expr_file_line_tuple); diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index a2d42e14592..f033b3400d0 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -23,10 +23,11 @@ use fold; use fold::*; use parse::{ParseSess, PResult, lexer}; use parse::parser::Parser; -use parse::token::{self, keywords}; +use parse::token; use print::pprust; use ptr::P; use std_inject; +use symbol::keywords; use tokenstream::{TokenTree, TokenStream}; use util::small_vector::SmallVector; use visit::Visitor; @@ -190,7 +191,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { self.cx.crate_root = std_inject::injected_crate_name(&krate); let mut module = ModuleData { - mod_path: vec![token::str_to_ident(&self.cx.ecfg.crate_name)], + mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], directory: PathBuf::from(self.cx.codemap().span_to_filename(krate.span)), }; module.directory.pop(); @@ -246,7 +247,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { self.cx.resolver.resolve_macro(scope, &mac.node.path, force) } InvocationKind::Attr { ref attr, .. } => { - let ident = ast::Ident::with_empty_ctxt(attr.name()); + let ident = Ident::with_empty_ctxt(attr.name()); let path = ast::Path::from_ident(attr.span, ident); self.cx.resolver.resolve_macro(scope, &path, force) } diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs index e323dd2f623..4fe57a8345e 100644 --- a/src/libsyntax/ext/placeholders.rs +++ b/src/libsyntax/ext/placeholders.rs @@ -13,8 +13,8 @@ use codemap::{DUMMY_SP, dummy_spanned}; use ext::base::ExtCtxt; use ext::expand::{Expansion, ExpansionKind}; use fold::*; -use parse::token::{intern, keywords}; use ptr::P; +use symbol::{Symbol, keywords}; use util::move_map::MoveMap; use util::small_vector::SmallVector; @@ -227,7 +227,7 @@ pub fn reconstructed_macro_rules(def: &ast::MacroDef) -> Expansion { span: DUMMY_SP, global: false, segments: vec![ast::PathSegment { - identifier: ast::Ident::with_empty_ctxt(intern("macro_rules")), + identifier: ast::Ident::with_empty_ctxt(Symbol::intern("macro_rules")), parameters: ast::PathParameters::none(), }], }, diff --git a/src/libsyntax/ext/proc_macro_shim.rs b/src/libsyntax/ext/proc_macro_shim.rs index dc3a01f41bc..21ce89a6dd5 100644 --- a/src/libsyntax/ext/proc_macro_shim.rs +++ b/src/libsyntax/ext/proc_macro_shim.rs @@ -66,6 +66,7 @@ pub mod prelude { pub use ast::Ident; pub use codemap::{DUMMY_SP, Span}; pub use ext::base::{ExtCtxt, MacResult}; - pub use parse::token::{self, Token, DelimToken, keywords, str_to_ident}; + pub use parse::token::{self, Token, DelimToken}; + pub use symbol::keywords; pub use tokenstream::{TokenTree, TokenStream}; } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index fa38f21e9b3..0bd018603d2 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -33,6 +33,7 @@ pub mod rt { use parse::{self, token, classify}; use ptr::P; use std::rc::Rc; + use symbol; use tokenstream::{self, TokenTree}; @@ -239,7 +240,7 @@ pub mod rt { impl ToTokens for str { fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> { let lit = ast::LitKind::Str( - token::intern_and_get_ident(self), ast::StrStyle::Cooked); + symbol::intern_and_get_ident(self), ast::StrStyle::Cooked); dummy_spanned(lit).to_tokens(cx) } } @@ -527,12 +528,12 @@ pub fn expand_quote_matcher(cx: &mut ExtCtxt, base::MacEager::expr(expanded) } -fn ids_ext(strs: Vec<String> ) -> Vec<ast::Ident> { - strs.iter().map(|str| str_to_ident(&(*str))).collect() +fn ids_ext(strs: Vec<String>) -> Vec<ast::Ident> { + strs.iter().map(|s| ast::Ident::from_str(s)).collect() } -fn id_ext(str: &str) -> ast::Ident { - str_to_ident(str) +fn id_ext(s: &str) -> ast::Ident { + ast::Ident::from_str(s) } // Lift an ident to the expr that evaluates to that ident. diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs index bda84cdaf39..7893ad839ea 100644 --- a/src/libsyntax/ext/source_util.rs +++ b/src/libsyntax/ext/source_util.rs @@ -17,6 +17,7 @@ use parse::token; use parse; use print::pprust; use ptr::P; +use symbol; use tokenstream; use util::small_vector::SmallVector; @@ -60,15 +61,14 @@ pub fn expand_file(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree]) let topmost = cx.expansion_cause(); let loc = cx.codemap().lookup_char_pos(topmost.lo); - let filename = token::intern_and_get_ident(&loc.file.name); + let filename = symbol::intern_and_get_ident(&loc.file.name); base::MacEager::expr(cx.expr_str(topmost, filename)) } pub fn expand_stringify(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree]) -> Box<base::MacResult+'static> { let s = pprust::tts_to_string(tts); - base::MacEager::expr(cx.expr_str(sp, - token::intern_and_get_ident(&s[..]))) + base::MacEager::expr(cx.expr_str(sp, symbol::intern_and_get_ident(&s))) } pub fn expand_mod(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree]) @@ -77,9 +77,7 @@ pub fn expand_mod(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree]) let mod_path = &cx.current_expansion.module.mod_path; let string = mod_path.iter().map(|x| x.to_string()).collect::<Vec<String>>().join("::"); - base::MacEager::expr(cx.expr_str( - sp, - token::intern_and_get_ident(&string[..]))) + base::MacEager::expr(cx.expr_str(sp, symbol::intern_and_get_ident(&string))) } /// include! : parse the given file as an expr @@ -144,7 +142,7 @@ pub fn expand_include_str(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenT // Add this input file to the code map to make it available as // dependency information let filename = format!("{}", file.display()); - let interned = token::intern_and_get_ident(&src[..]); + let interned = symbol::intern_and_get_ident(&src); cx.codemap().new_filemap_and_lines(&filename, None, &src); base::MacEager::expr(cx.expr_str(sp, interned)) diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 552d4de9617..4c37a0a71e2 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -20,9 +20,10 @@ use ext::tt::macro_parser::{parse, parse_failure_msg}; use parse::ParseSess; use parse::lexer::new_tt_reader; use parse::parser::{Parser, Restrictions}; -use parse::token::{self, gensym_ident, NtTT, Token}; +use parse::token::{self, NtTT, Token}; use parse::token::Token::*; use print; +use symbol::Symbol; use tokenstream::{self, TokenTree}; use std::collections::{HashMap}; @@ -187,16 +188,16 @@ impl IdentMacroExpander for MacroRulesExpander { /// Converts a `macro_rules!` invocation into a syntax extension. pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension { - let lhs_nm = gensym_ident("lhs"); - let rhs_nm = gensym_ident("rhs"); + let lhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("lhs")); + let rhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("rhs")); // The pattern that macro_rules matches. // The grammar for macro_rules! is: // $( $lhs:tt => $rhs:tt );+ // ...quasiquoting this would be nice. // These spans won't matter, anyways - let match_lhs_tok = MatchNt(lhs_nm, token::str_to_ident("tt")); - let match_rhs_tok = MatchNt(rhs_nm, token::str_to_ident("tt")); + let match_lhs_tok = MatchNt(lhs_nm, ast::Ident::from_str("tt")); + let match_rhs_tok = MatchNt(rhs_nm, ast::Ident::from_str("tt")); let argument_gram = vec![ TokenTree::Sequence(DUMMY_SP, Rc::new(tokenstream::SequenceRepetition { tts: vec![ diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 19f5e860b27..0f94018997c 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -33,7 +33,7 @@ use syntax_pos::Span; use errors::{DiagnosticBuilder, Handler}; use visit::{self, FnKind, Visitor}; use parse::ParseSess; -use parse::token::InternedString; +use symbol::InternedString; use std::ascii::AsciiExt; use std::env; diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index b6c87155d6b..44ce8668c54 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -22,8 +22,9 @@ use ast::*; use ast; use syntax_pos::Span; use codemap::{Spanned, respan}; -use parse::token::{self, keywords}; +use parse::token; use ptr::P; +use symbol::keywords; use tokenstream::*; use util::small_vector::SmallVector; use util::move_map::MoveMap; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 34280812421..15b93a4864b 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -83,7 +83,6 @@ pub mod diagnostics { pub mod diagnostic_list; pub mod util { - pub mod interner; pub mod lev_distance; pub mod node_count; pub mod parser; @@ -118,6 +117,7 @@ pub mod ptr; pub mod show_span; pub mod std_inject; pub mod str; +pub mod symbol; pub mod test; pub mod tokenstream; pub mod visit; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index cf48c445c80..7048be1478b 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -8,13 +8,14 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast; +use ast::{self, Ident}; use syntax_pos::{self, BytePos, CharPos, Pos, Span}; use codemap::CodeMap; use errors::{FatalError, Handler, DiagnosticBuilder}; use ext::tt::transcribe::tt_next_token; -use parse::token::{self, keywords, str_to_ident}; +use parse::token; use str::char_at; +use symbol::{Symbol, keywords}; use rustc_unicode::property::Pattern_White_Space; use std::borrow::Cow; @@ -350,13 +351,13 @@ impl<'a> StringReader<'a> { /// single-byte delimiter). pub fn name_from(&self, start: BytePos) -> ast::Name { debug!("taking an ident from {:?} to {:?}", start, self.pos); - self.with_str_from(start, token::intern) + self.with_str_from(start, Symbol::intern) } /// As name_from, with an explicit endpoint. pub fn name_from_to(&self, start: BytePos, end: BytePos) -> ast::Name { debug!("taking an ident from {:?} to {:?}", start, end); - self.with_str_from_to(start, end, token::intern) + self.with_str_from_to(start, end, Symbol::intern) } /// Calls `f` with a string slice of the source text spanning from `start` @@ -492,7 +493,7 @@ impl<'a> StringReader<'a> { if string == "_" { None } else { - Some(token::intern(string)) + Some(Symbol::intern(string)) } }) } @@ -540,7 +541,7 @@ impl<'a> StringReader<'a> { self.with_str_from(start_bpos, |string| { // comments with only more "/"s are not doc comments let tok = if is_doc_comment(string) { - token::DocComment(token::intern(string)) + token::DocComment(Symbol::intern(string)) } else { token::Comment }; @@ -669,7 +670,7 @@ impl<'a> StringReader<'a> { } else { string.into() }; - token::DocComment(token::intern(&string[..])) + token::DocComment(Symbol::intern(&string[..])) } else { token::Comment }; @@ -758,7 +759,7 @@ impl<'a> StringReader<'a> { self.err_span_(start_bpos, self.pos, "no valid digits found for number"); - return token::Integer(token::intern("0")); + return token::Integer(Symbol::intern("0")); } // might be a float, but don't be greedy if this is actually an @@ -1097,7 +1098,7 @@ impl<'a> StringReader<'a> { token::Underscore } else { // FIXME: perform NFKC normalization here. (Issue #2253) - token::Ident(str_to_ident(string)) + token::Ident(Ident::from_str(string)) } })); } @@ -1277,13 +1278,13 @@ impl<'a> StringReader<'a> { // expansion purposes. See #12512 for the gory details of why // this is necessary. let ident = self.with_str_from(start, |lifetime_name| { - str_to_ident(&format!("'{}", lifetime_name)) + Ident::from_str(&format!("'{}", lifetime_name)) }); // Conjure up a "keyword checking ident" to make sure that // the lifetime name is not a keyword. let keyword_checking_ident = self.with_str_from(start, |lifetime_name| { - str_to_ident(lifetime_name) + Ident::from_str(lifetime_name) }); let keyword_checking_token = &token::Ident(keyword_checking_ident); let last_bpos = self.pos; @@ -1310,7 +1311,7 @@ impl<'a> StringReader<'a> { let id = if valid { self.name_from(start) } else { - token::intern("0") + Symbol::intern("0") }; self.bump(); // advance ch past token let suffix = self.scan_optional_raw_name(); @@ -1352,7 +1353,7 @@ impl<'a> StringReader<'a> { let id = if valid { self.name_from(start_bpos + BytePos(1)) } else { - token::intern("??") + Symbol::intern("??") }; self.bump(); let suffix = self.scan_optional_raw_name(); @@ -1424,7 +1425,7 @@ impl<'a> StringReader<'a> { let id = if valid { self.name_from_to(content_start_bpos, content_end_bpos) } else { - token::intern("??") + Symbol::intern("??") }; let suffix = self.scan_optional_raw_name(); return Ok(token::Literal(token::StrRaw(id, hash_count), suffix)); @@ -1551,7 +1552,7 @@ impl<'a> StringReader<'a> { let id = if valid { self.name_from(start) } else { - token::intern("?") + Symbol::intern("?") }; self.bump(); // advance ch past token return token::Byte(id); @@ -1584,7 +1585,7 @@ impl<'a> StringReader<'a> { let id = if valid { self.name_from(start) } else { - token::intern("??") + Symbol::intern("??") }; self.bump(); return token::ByteStr(id); @@ -1700,11 +1701,11 @@ fn ident_continue(c: Option<char>) -> bool { mod tests { use super::*; + use ast::Ident; use syntax_pos::{BytePos, Span, NO_EXPANSION}; use codemap::CodeMap; use errors; use parse::token; - use parse::token::str_to_ident; use std::io; use std::rc::Rc; @@ -1732,7 +1733,7 @@ mod tests { &sh, "/* my source file */ fn main() { println!(\"zebra\"); }\n" .to_string()); - let id = str_to_ident("fn"); + let id = Ident::from_str("fn"); assert_eq!(string_reader.next_token().tok, token::Comment); assert_eq!(string_reader.next_token().tok, token::Whitespace); let tok1 = string_reader.next_token(); @@ -1813,7 +1814,7 @@ mod tests { let cm = Rc::new(CodeMap::new()); let sh = mk_sh(cm.clone()); assert_eq!(setup(&cm, &sh, "'a'".to_string()).next_token().tok, - token::Literal(token::Char(token::intern("a")), None)); + token::Literal(token::Char(Symbol::intern("a")), None)); } #[test] @@ -1821,7 +1822,7 @@ mod tests { let cm = Rc::new(CodeMap::new()); let sh = mk_sh(cm.clone()); assert_eq!(setup(&cm, &sh, "' '".to_string()).next_token().tok, - token::Literal(token::Char(token::intern(" ")), None)); + token::Literal(token::Char(Symbol::intern(" ")), None)); } #[test] @@ -1829,7 +1830,7 @@ mod tests { let cm = Rc::new(CodeMap::new()); let sh = mk_sh(cm.clone()); assert_eq!(setup(&cm, &sh, "'\\n'".to_string()).next_token().tok, - token::Literal(token::Char(token::intern("\\n")), None)); + token::Literal(token::Char(Symbol::intern("\\n")), None)); } #[test] @@ -1847,7 +1848,7 @@ mod tests { assert_eq!(setup(&cm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()) .next_token() .tok, - token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3), None)); + token::Literal(token::StrRaw(Symol::intern("\"#a\\b\x00c\""), 3), None)); } #[test] @@ -1857,11 +1858,11 @@ mod tests { macro_rules! test { ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ assert_eq!(setup(&cm, &sh, format!("{}suffix", $input)).next_token().tok, - token::Literal(token::$tok_type(token::intern($tok_contents)), - Some(token::intern("suffix")))); + token::Literal(token::$tok_type(Symbol::intern($tok_contents)), + Some(Symbol::intern("suffix")))); // with a whitespace separator: assert_eq!(setup(&cm, &sh, format!("{} suffix", $input)).next_token().tok, - token::Literal(token::$tok_type(token::intern($tok_contents)), + token::Literal(token::$tok_type(Symbol::intern($tok_contents)), None)); }} } @@ -1877,14 +1878,14 @@ mod tests { test!("1.0e10", Float, "1.0e10"); assert_eq!(setup(&cm, &sh, "2us".to_string()).next_token().tok, - token::Literal(token::Integer(token::intern("2")), - Some(token::intern("us")))); + token::Literal(token::Integer(Symbol::intern("2")), + Some(Symbol::intern("us")))); assert_eq!(setup(&cm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok, - token::Literal(token::StrRaw(token::intern("raw"), 3), - Some(token::intern("suffix")))); + token::Literal(token::StrRaw(Symbol::intern("raw"), 3), + Some(Symbol::intern("suffix")))); assert_eq!(setup(&cm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok, - token::Literal(token::ByteStrRaw(token::intern("raw"), 3), - Some(token::intern("suffix")))); + token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3), + Some(Symbol::intern("suffix")))); } #[test] @@ -1904,7 +1905,7 @@ mod tests { _ => panic!("expected a comment!"), } assert_eq!(lexer.next_token().tok, - token::Literal(token::Char(token::intern("a")), None)); + token::Literal(token::Char(Symbol::intern("a")), None)); } #[test] @@ -1917,6 +1918,6 @@ mod tests { assert_eq!(comment.sp, ::syntax_pos::mk_sp(BytePos(0), BytePos(7))); assert_eq!(lexer.next_token().tok, token::Whitespace); assert_eq!(lexer.next_token().tok, - token::DocComment(token::intern("/// test"))); + token::DocComment(Symbol::intern("/// test"))); } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 19e8b711ba4..a44f78b3c3a 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -16,9 +16,9 @@ use syntax_pos::{self, Span, FileMap}; use errors::{Handler, ColorConfig, DiagnosticBuilder}; use feature_gate::UnstableFeatures; use parse::parser::Parser; -use parse::token::InternedString; use ptr::P; use str::char_at; +use symbol::{self, InternedString}; use tokenstream; use std::cell::RefCell; @@ -372,7 +372,7 @@ fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool { s[1..].chars().all(|c| '0' <= c && c <= '9') } -fn filtered_float_lit(data: token::InternedString, suffix: Option<&str>, +fn filtered_float_lit(data: InternedString, suffix: Option<&str>, sd: &Handler, sp: Span) -> ast::LitKind { debug!("filtered_float_lit: {}, {:?}", data, suffix); match suffix.as_ref().map(|s| &**s) { @@ -400,7 +400,7 @@ pub fn float_lit(s: &str, suffix: Option<InternedString>, debug!("float_lit: {:?}, {:?}", s, suffix); // FIXME #2252: bounds checking float literals is deferred until trans let s = s.chars().filter(|&c| c != '_').collect::<String>(); - let data = token::intern_and_get_ident(&s); + let data = symbol::intern_and_get_ident(&s); filtered_float_lit(data, suffix.as_ref().map(|s| &**s), sd, sp) } @@ -530,7 +530,7 @@ pub fn integer_lit(s: &str, 2 => sd.span_err(sp, "binary float literal is not supported"), _ => () } - let ident = token::intern_and_get_ident(&s); + let ident = symbol::intern_and_get_ident(&s); return filtered_float_lit(ident, Some(&suf), sd, sp) } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 98ce00c7d38..13c701795a8 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -48,13 +48,14 @@ use parse::classify; use parse::common::SeqSep; use parse::lexer::{Reader, TokenAndSpan}; use parse::obsolete::ObsoleteSyntax; -use parse::token::{self, intern, keywords, MatchNt, SubstNt, InternedString}; +use parse::token::{self, MatchNt, SubstNt}; use parse::{new_sub_parser_from_file, ParseSess}; use util::parser::{AssocOp, Fixity}; use print::pprust; use ptr::P; use parse::PResult; use tokenstream::{self, Delimited, SequenceRepetition, TokenTree}; +use symbol::{self, Symbol, keywords, InternedString}; use util::ThinVec; use std::collections::HashSet; @@ -1537,13 +1538,13 @@ impl<'a> Parser<'a> { token::Str_(s) => { (true, - LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())), + LitKind::Str(symbol::intern_and_get_ident(&parse::str_lit(&s.as_str())), ast::StrStyle::Cooked)) } token::StrRaw(s, n) => { (true, LitKind::Str( - token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), + symbol::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), ast::StrStyle::Raw(n))) } token::ByteStr(i) => @@ -2627,7 +2628,7 @@ impl<'a> Parser<'a> { }))); } else if self.token.is_keyword(keywords::Crate) { let ident = match self.token { - token::Ident(id) => ast::Ident { name: token::intern("$crate"), ..id }, + token::Ident(id) => ast::Ident { name: Symbol::intern("$crate"), ..id }, _ => unreachable!(), }; self.bump(); @@ -4835,7 +4836,7 @@ impl<'a> Parser<'a> { Visibility::Inherited => (), _ => { let is_macro_rules: bool = match self.token { - token::Ident(sid) => sid.name == intern("macro_rules"), + token::Ident(sid) => sid.name == Symbol::intern("macro_rules"), _ => false, }; if is_macro_rules { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 4aaa028ef75..8ac39dd462e 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -16,13 +16,10 @@ pub use self::Token::*; use ast::{self}; use ptr::P; -use util::interner::Interner; +use symbol::keywords; use tokenstream; -use serialize::{Decodable, Decoder, Encodable, Encoder}; -use std::cell::RefCell; use std::fmt; -use std::ops::Deref; use std::rc::Rc; #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] @@ -335,266 +332,3 @@ impl fmt::Debug for Nonterminal { } } } - -// In this macro, there is the requirement that the name (the number) must be monotonically -// increasing by one in the special identifiers, starting at 0; the same holds for the keywords, -// except starting from the next number instead of zero. -macro_rules! declare_keywords {( - $( ($index: expr, $konst: ident, $string: expr) )* -) => { - pub mod keywords { - use ast; - #[derive(Clone, Copy, PartialEq, Eq)] - pub struct Keyword { - ident: ast::Ident, - } - impl Keyword { - #[inline] pub fn ident(self) -> ast::Ident { self.ident } - #[inline] pub fn name(self) -> ast::Name { self.ident.name } - } - $( - #[allow(non_upper_case_globals)] - pub const $konst: Keyword = Keyword { - ident: ast::Ident::with_empty_ctxt(ast::Name($index)) - }; - )* - } - - fn mk_fresh_ident_interner() -> IdentInterner { - Interner::prefill(&[$($string,)*]) - } -}} - -// NB: leaving holes in the ident table is bad! a different ident will get -// interned with the id from the hole, but it will be between the min and max -// of the reserved words, and thus tagged as "reserved". -// After modifying this list adjust `is_strict_keyword`/`is_reserved_keyword`, -// this should be rarely necessary though if the keywords are kept in alphabetic order. -declare_keywords! { - // Invalid identifier - (0, Invalid, "") - - // Strict keywords used in the language. - (1, As, "as") - (2, Box, "box") - (3, Break, "break") - (4, Const, "const") - (5, Continue, "continue") - (6, Crate, "crate") - (7, Else, "else") - (8, Enum, "enum") - (9, Extern, "extern") - (10, False, "false") - (11, Fn, "fn") - (12, For, "for") - (13, If, "if") - (14, Impl, "impl") - (15, In, "in") - (16, Let, "let") - (17, Loop, "loop") - (18, Match, "match") - (19, Mod, "mod") - (20, Move, "move") - (21, Mut, "mut") - (22, Pub, "pub") - (23, Ref, "ref") - (24, Return, "return") - (25, SelfValue, "self") - (26, SelfType, "Self") - (27, Static, "static") - (28, Struct, "struct") - (29, Super, "super") - (30, Trait, "trait") - (31, True, "true") - (32, Type, "type") - (33, Unsafe, "unsafe") - (34, Use, "use") - (35, Where, "where") - (36, While, "while") - - // Keywords reserved for future use. - (37, Abstract, "abstract") - (38, Alignof, "alignof") - (39, Become, "become") - (40, Do, "do") - (41, Final, "final") - (42, Macro, "macro") - (43, Offsetof, "offsetof") - (44, Override, "override") - (45, Priv, "priv") - (46, Proc, "proc") - (47, Pure, "pure") - (48, Sizeof, "sizeof") - (49, Typeof, "typeof") - (50, Unsized, "unsized") - (51, Virtual, "virtual") - (52, Yield, "yield") - - // Weak keywords, have special meaning only in specific contexts. - (53, Default, "default") - (54, StaticLifetime, "'static") - (55, Union, "union") -} - -// looks like we can get rid of this completely... -pub type IdentInterner = Interner; - -// if an interner exists in TLS, return it. Otherwise, prepare a -// fresh one. -// FIXME(eddyb) #8726 This should probably use a thread-local reference. -pub fn with_ident_interner<T, F: FnOnce(&mut IdentInterner) -> T>(f: F) -> T { - thread_local!(static KEY: RefCell<IdentInterner> = { - RefCell::new(mk_fresh_ident_interner()) - }); - KEY.with(|interner| f(&mut *interner.borrow_mut())) -} - -/// Reset the ident interner to its initial state. -pub fn reset_ident_interner() { - with_ident_interner(|interner| *interner = mk_fresh_ident_interner()); -} - -/// Represents a string stored in the thread-local interner. Because the -/// interner lives for the life of the thread, this can be safely treated as an -/// immortal string, as long as it never crosses between threads. -/// -/// FIXME(pcwalton): You must be careful about what you do in the destructors -/// of objects stored in TLS, because they may run after the interner is -/// destroyed. In particular, they must not access string contents. This can -/// be fixed in the future by just leaking all strings until thread death -/// somehow. -#[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)] -pub struct InternedString { - string: Rc<str>, -} - -impl InternedString { - #[inline] - pub fn new(string: &'static str) -> InternedString { - InternedString { - string: Rc::__from_str(string), - } - } - - #[inline] - pub fn new_from_name(name: ast::Name) -> InternedString { - with_ident_interner(|interner| InternedString { string: interner.get(name) }) - } -} - -impl Deref for InternedString { - type Target = str; - - fn deref(&self) -> &str { &self.string } -} - -impl fmt::Debug for InternedString { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self.string, f) - } -} - -impl fmt::Display for InternedString { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.string, f) - } -} - -impl<'a> PartialEq<&'a str> for InternedString { - #[inline(always)] - fn eq(&self, other: & &'a str) -> bool { - PartialEq::eq(&self.string[..], *other) - } - #[inline(always)] - fn ne(&self, other: & &'a str) -> bool { - PartialEq::ne(&self.string[..], *other) - } -} - -impl<'a> PartialEq<InternedString> for &'a str { - #[inline(always)] - fn eq(&self, other: &InternedString) -> bool { - PartialEq::eq(*self, &other.string[..]) - } - #[inline(always)] - fn ne(&self, other: &InternedString) -> bool { - PartialEq::ne(*self, &other.string[..]) - } -} - -impl PartialEq<str> for InternedString { - #[inline(always)] - fn eq(&self, other: &str) -> bool { - PartialEq::eq(&self.string[..], other) - } - #[inline(always)] - fn ne(&self, other: &str) -> bool { - PartialEq::ne(&self.string[..], other) - } -} - -impl PartialEq<InternedString> for str { - #[inline(always)] - fn eq(&self, other: &InternedString) -> bool { - PartialEq::eq(self, &other.string[..]) - } - #[inline(always)] - fn ne(&self, other: &InternedString) -> bool { - PartialEq::ne(self, &other.string[..]) - } -} - -impl Decodable for InternedString { - fn decode<D: Decoder>(d: &mut D) -> Result<InternedString, D::Error> { - Ok(intern(&d.read_str()?).as_str()) - } -} - -impl Encodable for InternedString { - fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_str(&self.string) - } -} - -/// Interns and returns the string contents of an identifier, using the -/// thread-local interner. -#[inline] -pub fn intern_and_get_ident(s: &str) -> InternedString { - intern(s).as_str() -} - -/// Maps a string to its interned representation. -#[inline] -pub fn intern(s: &str) -> ast::Name { - with_ident_interner(|interner| interner.intern(s)) -} - -/// gensym's a new usize, using the current interner. -#[inline] -pub fn gensym(s: &str) -> ast::Name { - with_ident_interner(|interner| interner.gensym(s)) -} - -/// Maps a string to an identifier with an empty syntax context. -#[inline] -pub fn str_to_ident(s: &str) -> ast::Ident { - ast::Ident::with_empty_ctxt(intern(s)) -} - -/// Maps a string to a gensym'ed identifier. -#[inline] -pub fn gensym_ident(s: &str) -> ast::Ident { - ast::Ident::with_empty_ctxt(gensym(s)) -} - -// create a fresh name that maps to the same string as the old one. -// note that this guarantees that str_ptr_eq(ident_to_string(src),interner_get(fresh_name(src))); -// that is, that the new name and the old one are connected to ptr_eq strings. -pub fn fresh_name(src: ast::Ident) -> ast::Name { - with_ident_interner(|interner| interner.gensym_copy(src.name)) - // following: debug version. Could work in final except that it's incompatible with - // good error messages and uses of struct names in ambiguous could-be-binding - // locations. Also definitely destroys the guarantee given above about ptr_eq. - /*let num = rand::thread_rng().gen_uint_range(0,0xffff); - gensym(format!("{}_{}",ident_to_string(src),num))*/ -} diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index da766d17672..52ff17d1c12 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -19,7 +19,7 @@ use attr; use codemap::{self, CodeMap}; use syntax_pos::{self, BytePos}; use errors; -use parse::token::{self, keywords, BinOpToken, Token}; +use parse::token::{self, BinOpToken, Token}; use parse::lexer::comments; use parse; use print::pp::{self, break_offset, word, space, zerobreak, hardbreak}; @@ -27,6 +27,7 @@ use print::pp::{Breaks, eof}; use print::pp::Breaks::{Consistent, Inconsistent}; use ptr::P; use std_inject; +use symbol::{Symbol, keywords}; use tokenstream::{self, TokenTree}; use std::ascii; @@ -119,13 +120,13 @@ pub fn print_crate<'a>(cm: &'a CodeMap, // of the feature gate, so we fake them up here. // #![feature(prelude_import)] - let prelude_import_meta = attr::mk_list_word_item(token::intern("prelude_import")); - let list = attr::mk_list_item(token::intern("feature"), vec![prelude_import_meta]); + let prelude_import_meta = attr::mk_list_word_item(Symbol::intern("prelude_import")); + let list = attr::mk_list_item(Symbol::intern("feature"), vec![prelude_import_meta]); let fake_attr = attr::mk_attr_inner(attr::mk_attr_id(), list); try!(s.print_attribute(&fake_attr)); // #![no_std] - let no_std_meta = attr::mk_word_item(token::intern("no_std")); + let no_std_meta = attr::mk_word_item(Symbol::intern("no_std")); let fake_attr = attr::mk_attr_inner(attr::mk_attr_id(), no_std_meta); try!(s.print_attribute(&fake_attr)); } diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs index 166d95911b9..6a291ad9c40 100644 --- a/src/libsyntax/std_inject.rs +++ b/src/libsyntax/std_inject.rs @@ -10,10 +10,10 @@ use ast; use attr; +use symbol::{Symbol, keywords}; use syntax_pos::{DUMMY_SP, Span}; use codemap::{self, ExpnInfo, NameAndSpan, MacroAttribute}; -use parse::token::{intern, keywords}; -use parse::{token, ParseSess}; +use parse::ParseSess; use ptr::P; /// Craft a span that will be ignored by the stability lint's @@ -23,7 +23,7 @@ fn ignored_span(sess: &ParseSess, sp: Span) -> Span { let info = ExpnInfo { call_site: DUMMY_SP, callee: NameAndSpan { - format: MacroAttribute(intern("std_inject")), + format: MacroAttribute(Symbol::intern("std_inject")), span: None, allow_internal_unstable: true, } @@ -53,14 +53,14 @@ pub fn maybe_inject_crates_ref(sess: &ParseSess, None => return krate, }; - let crate_name = token::intern(&alt_std_name.unwrap_or(name.to_string())); + let crate_name = Symbol::intern(&alt_std_name.unwrap_or(name.to_string())); krate.module.items.insert(0, P(ast::Item { attrs: vec![attr::mk_attr_outer(attr::mk_attr_id(), - attr::mk_word_item(token::intern("macro_use")))], + attr::mk_word_item(Symbol::intern("macro_use")))], vis: ast::Visibility::Inherited, node: ast::ItemKind::ExternCrate(Some(crate_name)), - ident: token::str_to_ident(name), + ident: ast::Ident::from_str(name), id: ast::DUMMY_NODE_ID, span: DUMMY_SP, })); @@ -70,7 +70,7 @@ pub fn maybe_inject_crates_ref(sess: &ParseSess, attrs: vec![ast::Attribute { style: ast::AttrStyle::Outer, value: ast::MetaItem { - name: token::intern("prelude_import"), + name: Symbol::intern("prelude_import"), node: ast::MetaItemKind::Word, span: span, }, @@ -82,7 +82,7 @@ pub fn maybe_inject_crates_ref(sess: &ParseSess, node: ast::ItemKind::Use(P(codemap::dummy_spanned(ast::ViewPathGlob(ast::Path { global: false, segments: vec![name, "prelude", "v1"].into_iter().map(|name| ast::PathSegment { - identifier: token::str_to_ident(name), + identifier: ast::Ident::from_str(name), parameters: ast::PathParameters::none(), }).collect(), span: span, diff --git a/src/libsyntax/symbol.rs b/src/libsyntax/symbol.rs new file mode 100644 index 00000000000..9620b8412b4 --- /dev/null +++ b/src/libsyntax/symbol.rs @@ -0,0 +1,339 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! An "interner" is a data structure that associates values with usize tags and +//! allows bidirectional lookup; i.e. given a value, one can easily find the +//! type, and vice versa. + +use serialize::{Decodable, Decoder, Encodable, Encoder}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::fmt; +use std::rc::Rc; + +/// A symbol is an interned or gensymed string. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Symbol(u32); + +impl Symbol { + /// Maps a string to its interned representation. + pub fn intern(string: &str) -> Self { + with_interner(|interner| interner.intern(string)) + } + + /// gensym's a new usize, using the current interner. + pub fn gensym(string: &str) -> Self { + with_interner(|interner| interner.gensym(string)) + } + + pub fn as_str(self) -> InternedString { + with_interner(|interner| InternedString { string: interner.get(self) }) + } + + pub fn as_u32(self) -> u32 { + self.0 + } +} + +impl fmt::Debug for Symbol { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}({})", self, self.0) + } +} + +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.as_str(), f) + } +} + +impl Encodable for Symbol { + fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { + s.emit_str(&self.as_str()) + } +} + +impl Decodable for Symbol { + fn decode<D: Decoder>(d: &mut D) -> Result<Symbol, D::Error> { + Ok(Symbol::intern(&d.read_str()?)) + } +} + +impl<'a> PartialEq<&'a str> for Symbol { + fn eq(&self, other: &&str) -> bool { + *self.as_str() == **other + } +} + +#[derive(Default)] +pub struct Interner { + names: HashMap<Rc<str>, Symbol>, + strings: Vec<Rc<str>>, +} + +impl Interner { + pub fn new() -> Self { + Interner::default() + } + + fn prefill(init: &[&str]) -> Self { + let mut this = Interner::new(); + for &string in init { + this.intern(string); + } + this + } + + pub fn intern(&mut self, string: &str) -> Symbol { + if let Some(&name) = self.names.get(string) { + return name; + } + + let name = Symbol(self.strings.len() as u32); + let string = Rc::__from_str(string); + self.strings.push(string.clone()); + self.names.insert(string, name); + name + } + + fn gensym(&mut self, string: &str) -> Symbol { + let gensym = Symbol(self.strings.len() as u32); + // leave out of `names` to avoid colliding + self.strings.push(Rc::__from_str(string)); + gensym + } + + pub fn get(&self, name: Symbol) -> Rc<str> { + self.strings[name.0 as usize].clone() + } +} + +// In this macro, there is the requirement that the name (the number) must be monotonically +// increasing by one in the special identifiers, starting at 0; the same holds for the keywords, +// except starting from the next number instead of zero. +macro_rules! declare_keywords {( + $( ($index: expr, $konst: ident, $string: expr) )* +) => { + pub mod keywords { + use ast; + #[derive(Clone, Copy, PartialEq, Eq)] + pub struct Keyword { + ident: ast::Ident, + } + impl Keyword { + #[inline] pub fn ident(self) -> ast::Ident { self.ident } + #[inline] pub fn name(self) -> ast::Name { self.ident.name } + } + $( + #[allow(non_upper_case_globals)] + pub const $konst: Keyword = Keyword { + ident: ast::Ident::with_empty_ctxt(ast::Name($index)) + }; + )* + } + + impl Interner { + fn fresh() -> Self { + Interner::prefill(&[$($string,)*]) + } + } +}} + +// NB: leaving holes in the ident table is bad! a different ident will get +// interned with the id from the hole, but it will be between the min and max +// of the reserved words, and thus tagged as "reserved". +// After modifying this list adjust `is_strict_keyword`/`is_reserved_keyword`, +// this should be rarely necessary though if the keywords are kept in alphabetic order. +declare_keywords! { + // Invalid identifier + (0, Invalid, "") + + // Strict keywords used in the language. + (1, As, "as") + (2, Box, "box") + (3, Break, "break") + (4, Const, "const") + (5, Continue, "continue") + (6, Crate, "crate") + (7, Else, "else") + (8, Enum, "enum") + (9, Extern, "extern") + (10, False, "false") + (11, Fn, "fn") + (12, For, "for") + (13, If, "if") + (14, Impl, "impl") + (15, In, "in") + (16, Let, "let") + (17, Loop, "loop") + (18, Match, "match") + (19, Mod, "mod") + (20, Move, "move") + (21, Mut, "mut") + (22, Pub, "pub") + (23, Ref, "ref") + (24, Return, "return") + (25, SelfValue, "self") + (26, SelfType, "Self") + (27, Static, "static") + (28, Struct, "struct") + (29, Super, "super") + (30, Trait, "trait") + (31, True, "true") + (32, Type, "type") + (33, Unsafe, "unsafe") + (34, Use, "use") + (35, Where, "where") + (36, While, "while") + + // Keywords reserved for future use. + (37, Abstract, "abstract") + (38, Alignof, "alignof") + (39, Become, "become") + (40, Do, "do") + (41, Final, "final") + (42, Macro, "macro") + (43, Offsetof, "offsetof") + (44, Override, "override") + (45, Priv, "priv") + (46, Proc, "proc") + (47, Pure, "pure") + (48, Sizeof, "sizeof") + (49, Typeof, "typeof") + (50, Unsized, "unsized") + (51, Virtual, "virtual") + (52, Yield, "yield") + + // Weak keywords, have special meaning only in specific contexts. + (53, Default, "default") + (54, StaticLifetime, "'static") + (55, Union, "union") +} + +// If an interner exists in TLS, return it. Otherwise, prepare a fresh one. +fn with_interner<T, F: FnOnce(&mut Interner) -> T>(f: F) -> T { + thread_local!(static INTERNER: RefCell<Interner> = { + RefCell::new(Interner::fresh()) + }); + INTERNER.with(|interner| f(&mut *interner.borrow_mut())) +} + +/// Reset the ident interner to its initial state. +pub fn reset_interner() { + with_interner(|interner| *interner = Interner::fresh()); +} + +/// Represents a string stored in the thread-local interner. Because the +/// interner lives for the life of the thread, this can be safely treated as an +/// immortal string, as long as it never crosses between threads. +/// +/// FIXME(pcwalton): You must be careful about what you do in the destructors +/// of objects stored in TLS, because they may run after the interner is +/// destroyed. In particular, they must not access string contents. This can +/// be fixed in the future by just leaking all strings until thread death +/// somehow. +#[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)] +pub struct InternedString { + string: Rc<str>, +} + +impl InternedString { + pub fn new(string: &'static str) -> InternedString { + InternedString { + string: Rc::__from_str(string), + } + } +} + +impl ::std::ops::Deref for InternedString { + type Target = str; + fn deref(&self) -> &str { &self.string } +} + +impl fmt::Debug for InternedString { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.string, f) + } +} + +impl fmt::Display for InternedString { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.string, f) + } +} + +impl<'a> PartialEq<&'a str> for InternedString { + fn eq(&self, other: & &'a str) -> bool { + PartialEq::eq(&self.string[..], *other) + } +} + +impl<'a> PartialEq<InternedString> for &'a str { + fn eq(&self, other: &InternedString) -> bool { + PartialEq::eq(*self, &other.string[..]) + } +} + +impl PartialEq<str> for InternedString { + fn eq(&self, other: &str) -> bool { + PartialEq::eq(&self.string[..], other) + } +} + +impl PartialEq<InternedString> for str { + fn eq(&self, other: &InternedString) -> bool { + PartialEq::eq(self, &other.string[..]) + } +} + +impl Decodable for InternedString { + fn decode<D: Decoder>(d: &mut D) -> Result<InternedString, D::Error> { + Ok(Symbol::intern(&d.read_str()?).as_str()) + } +} + +impl Encodable for InternedString { + fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { + s.emit_str(&self.string) + } +} + +/// Interns and returns the string contents of an identifier, using the +/// thread-local interner. +#[inline] +pub fn intern_and_get_ident(s: &str) -> InternedString { + Symbol::intern(s).as_str() +} + +#[cfg(test)] +mod tests { + use super::*; + use ast::Name; + + #[test] + fn interner_tests() { + let mut i: Interner = Interner::new(); + // first one is zero: + assert_eq!(i.intern("dog"), Name(0)); + // re-use gets the same entry: + assert_eq!(i.intern ("dog"), Name(0)); + // different string gets a different #: + assert_eq!(i.intern("cat"), Name(1)); + assert_eq!(i.intern("cat"), Name(1)); + // dog is still at zero + assert_eq!(i.intern("dog"), Name(0)); + // gensym gets 3 + assert_eq!(i.gensym("zebra"), Name(2)); + // gensym of same string gets new number : + assert_eq!(i.gensym("zebra"), Name(3)); + // gensym of *existing* string gets new number: + assert_eq!(i.gensym("dog"), Name(4)); + } +} diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs index 194bd4a9085..271de16e7ac 100644 --- a/src/libsyntax/test.rs +++ b/src/libsyntax/test.rs @@ -34,11 +34,11 @@ use ext::expand::ExpansionConfig; use fold::Folder; use util::move_map::MoveMap; use fold; -use parse::token::{intern, keywords, InternedString}; use parse::{token, ParseSess}; use print::pprust; -use ast; +use ast::{self, Ident}; use ptr::P; +use symbol::{self, Symbol, keywords, InternedString}; use util::small_vector::SmallVector; enum ShouldPanic { @@ -48,7 +48,7 @@ enum ShouldPanic { struct Test { span: Span, - path: Vec<ast::Ident> , + path: Vec<Ident> , bench: bool, ignore: bool, should_panic: ShouldPanic @@ -57,14 +57,14 @@ struct Test { struct TestCtxt<'a> { sess: &'a ParseSess, span_diagnostic: &'a errors::Handler, - path: Vec<ast::Ident>, + path: Vec<Ident>, ext_cx: ExtCtxt<'a>, testfns: Vec<Test>, reexport_test_harness_main: Option<InternedString>, is_test_crate: bool, // top-level re-export submodule, filled out after folding is finished - toplevel_reexport: Option<ast::Ident>, + toplevel_reexport: Option<Ident>, } // Traverse the crate, collecting all the test functions, eliding any @@ -91,10 +91,10 @@ pub fn modify_for_testing(sess: &ParseSess, struct TestHarnessGenerator<'a> { cx: TestCtxt<'a>, - tests: Vec<ast::Ident>, + tests: Vec<Ident>, // submodule name, gensym'd identifier for re-exports - tested_submods: Vec<(ast::Ident, ast::Ident)>, + tested_submods: Vec<(Ident, Ident)>, } impl<'a> fold::Folder for TestHarnessGenerator<'a> { @@ -191,8 +191,8 @@ impl fold::Folder for EntryPointCleaner { EntryPointType::MainAttr | EntryPointType::Start => folded.map(|ast::Item {id, ident, attrs, node, vis, span}| { - let allow_str = token::intern("allow"); - let dead_code_str = token::intern("dead_code"); + let allow_str = Symbol::intern("allow"); + let dead_code_str = Symbol::intern("dead_code"); let word_vec = vec![attr::mk_list_word_item(dead_code_str)]; let allow_dead_code_item = attr::mk_list_item(allow_str, word_vec); let allow_dead_code = attr::mk_attr_outer(attr::mk_attr_id(), @@ -222,15 +222,18 @@ impl fold::Folder for EntryPointCleaner { fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { mac } } -fn mk_reexport_mod(cx: &mut TestCtxt, parent: ast::NodeId, tests: Vec<ast::Ident>, - tested_submods: Vec<(ast::Ident, ast::Ident)>) -> (P<ast::Item>, ast::Ident) { - let super_ = token::str_to_ident("super"); +fn mk_reexport_mod(cx: &mut TestCtxt, + parent: ast::NodeId, + tests: Vec<Ident>, + tested_submods: Vec<(Ident, Ident)>) + -> (P<ast::Item>, Ident) { + let super_ = Ident::from_str("super"); // Generate imports with `#[allow(private_in_public)]` to work around issue #36768. let allow_private_in_public = cx.ext_cx.attribute(DUMMY_SP, cx.ext_cx.meta_list( DUMMY_SP, - token::intern("allow"), - vec![cx.ext_cx.meta_list_item_word(DUMMY_SP, token::intern("private_in_public"))], + Symbol::intern("allow"), + vec![cx.ext_cx.meta_list_item_word(DUMMY_SP, Symbol::intern("private_in_public"))], )); let items = tests.into_iter().map(|r| { cx.ext_cx.item_use_simple(DUMMY_SP, ast::Visibility::Public, @@ -247,7 +250,7 @@ fn mk_reexport_mod(cx: &mut TestCtxt, parent: ast::NodeId, tests: Vec<ast::Ident items: items, }; - let sym = token::gensym_ident("__test_reexports"); + let sym = Ident::with_empty_ctxt(Symbol::gensym("__test_reexports")); let parent = if parent == ast::DUMMY_NODE_ID { ast::CRATE_NODE_ID } else { parent }; cx.ext_cx.current_expansion.mark = cx.ext_cx.resolver.get_module_scope(parent); let it = cx.ext_cx.monotonic_expander().fold_item(P(ast::Item { @@ -286,7 +289,7 @@ fn generate_test_harness(sess: &ParseSess, cx.ext_cx.bt_push(ExpnInfo { call_site: DUMMY_SP, callee: NameAndSpan { - format: MacroAttribute(intern("test")), + format: MacroAttribute(Symbol::intern("test")), span: None, allow_internal_unstable: false, } @@ -306,7 +309,7 @@ fn ignored_span(cx: &TestCtxt, sp: Span) -> Span { let info = ExpnInfo { call_site: DUMMY_SP, callee: NameAndSpan { - format: MacroAttribute(intern("test")), + format: MacroAttribute(Symbol::intern("test")), span: None, allow_internal_unstable: true, } @@ -456,7 +459,7 @@ mod __test { */ fn mk_std(cx: &TestCtxt) -> P<ast::Item> { - let id_test = token::str_to_ident("test"); + let id_test = Ident::from_str("test"); let (vi, vis, ident) = if cx.is_test_crate { (ast::ItemKind::Use( P(nospan(ast::ViewPathSimple(id_test, @@ -487,16 +490,17 @@ fn mk_main(cx: &mut TestCtxt) -> P<ast::Item> { let ecx = &cx.ext_cx; // test::test_main_static - let test_main_path = ecx.path(sp, vec![token::str_to_ident("test"), - token::str_to_ident("test_main_static")]); + let test_main_path = + ecx.path(sp, vec![Ident::from_str("test"), Ident::from_str("test_main_static")]); + // test::test_main_static(...) let test_main_path_expr = ecx.expr_path(test_main_path); - let tests_ident_expr = ecx.expr_ident(sp, token::str_to_ident("TESTS")); + let tests_ident_expr = ecx.expr_ident(sp, Ident::from_str("TESTS")); let call_test_main = ecx.expr_call(sp, test_main_path_expr, vec![tests_ident_expr]); let call_test_main = ecx.stmt_expr(call_test_main); // #![main] - let main_meta = ecx.meta_word(sp, token::intern("main")); + let main_meta = ecx.meta_word(sp, Symbol::intern("main")); let main_attr = ecx.attribute(sp, main_meta); // pub fn main() { ... } let main_ret_ty = ecx.ty(sp, ast::TyKind::Tup(vec![])); @@ -506,7 +510,7 @@ fn mk_main(cx: &mut TestCtxt) -> P<ast::Item> { dummy_spanned(ast::Constness::NotConst), ::abi::Abi::Rust, ast::Generics::default(), main_body); let main = P(ast::Item { - ident: token::str_to_ident("main"), + ident: Ident::from_str("main"), attrs: vec![main_attr], id: ast::DUMMY_NODE_ID, node: main, @@ -533,7 +537,7 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P<ast::Item>, Option<P<ast::Item>>) { items: vec![import, mainfn, tests], }; let item_ = ast::ItemKind::Mod(testmod); - let mod_ident = token::gensym_ident("__test"); + let mod_ident = Ident::with_empty_ctxt(Symbol::gensym("__test")); let mut expander = cx.ext_cx.monotonic_expander(); let item = expander.fold_item(P(ast::Item { @@ -546,11 +550,11 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P<ast::Item>, Option<P<ast::Item>>) { })).pop().unwrap(); let reexport = cx.reexport_test_harness_main.as_ref().map(|s| { // building `use <ident> = __test::main` - let reexport_ident = token::str_to_ident(&s); + let reexport_ident = Ident::from_str(&s); let use_path = nospan(ast::ViewPathSimple(reexport_ident, - path_node(vec![mod_ident, token::str_to_ident("main")]))); + path_node(vec![mod_ident, Ident::from_str("main")]))); expander.fold_item(P(ast::Item { id: ast::DUMMY_NODE_ID, @@ -571,7 +575,7 @@ fn nospan<T>(t: T) -> codemap::Spanned<T> { codemap::Spanned { node: t, span: DUMMY_SP } } -fn path_node(ids: Vec<ast::Ident> ) -> ast::Path { +fn path_node(ids: Vec<Ident>) -> ast::Path { ast::Path { span: DUMMY_SP, global: false, @@ -582,7 +586,7 @@ fn path_node(ids: Vec<ast::Ident> ) -> ast::Path { } } -fn path_name_i(idents: &[ast::Ident]) -> String { +fn path_name_i(idents: &[Ident]) -> String { // FIXME: Bad copies (#2543 -- same for everything else that says "bad") idents.iter().map(|i| i.to_string()).collect::<Vec<String>>().join("::") } @@ -660,7 +664,7 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> { // path to the #[test] function: "foo::bar::baz" let path_string = path_name_i(&path[..]); - let name_expr = ecx.expr_str(span, token::intern_and_get_ident(&path_string[..])); + let name_expr = ecx.expr_str(span, symbol::intern_and_get_ident(&path_string[..])); // self::test::StaticTestName($name_expr) let name_expr = ecx.expr_call(span, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 9ef6c07e489..7ca4650a3ae 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -34,6 +34,7 @@ use parse::lexer; use parse; use parse::token::{self, Token, Lit, Nonterminal}; use print::pprust; +use symbol::{self, Symbol}; use std::fmt; use std::iter::*; @@ -173,10 +174,10 @@ impl TokenTree { TokenTree::Delimited(sp, Rc::new(Delimited { delim: token::Bracket, open_span: sp, - tts: vec![TokenTree::Token(sp, token::Ident(token::str_to_ident("doc"))), + tts: vec![TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))), TokenTree::Token(sp, token::Eq), TokenTree::Token(sp, token::Literal( - token::StrRaw(token::intern(&stripped), num_of_hashes), None))], + token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))], close_span: sp, })) } @@ -295,7 +296,7 @@ impl TokenTree { pub fn maybe_str(&self) -> Option<ast::Lit> { match *self { TokenTree::Token(sp, Token::Literal(Lit::Str_(s), _)) => { - let l = LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())), + let l = LitKind::Str(symbol::intern_and_get_ident(&parse::str_lit(&s.as_str())), ast::StrStyle::Cooked); Some(Spanned { node: l, @@ -303,7 +304,7 @@ impl TokenTree { }) } TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => { - let l = LitKind::Str(token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), + let l = LitKind::Str(symbol::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), ast::StrStyle::Raw(n)); Some(Spanned { node: l, diff --git a/src/libsyntax/util/interner.rs b/src/libsyntax/util/interner.rs deleted file mode 100644 index f56c6cedcd1..00000000000 --- a/src/libsyntax/util/interner.rs +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2012 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! An "interner" is a data structure that associates values with usize tags and -//! allows bidirectional lookup; i.e. given a value, one can easily find the -//! type, and vice versa. - -use ast::Name; - -use std::collections::HashMap; -use std::rc::Rc; - -#[derive(Default)] -pub struct Interner { - names: HashMap<Rc<str>, Name>, - strings: Vec<Rc<str>>, -} - -/// When traits can extend traits, we should extend index<Name,T> to get [] -impl Interner { - pub fn new() -> Self { - Interner::default() - } - - pub fn prefill(init: &[&str]) -> Self { - let mut this = Interner::new(); - for &string in init { - this.intern(string); - } - this - } - - pub fn intern(&mut self, string: &str) -> Name { - if let Some(&name) = self.names.get(string) { - return name; - } - - let name = Name(self.strings.len() as u32); - let string = Rc::__from_str(string); - self.strings.push(string.clone()); - self.names.insert(string, name); - name - } - - pub fn gensym(&mut self, string: &str) -> Name { - let gensym = Name(self.strings.len() as u32); - // leave out of `names` to avoid colliding - self.strings.push(Rc::__from_str(string)); - gensym - } - - /// Create a gensym with the same name as an existing entry. - pub fn gensym_copy(&mut self, name: Name) -> Name { - let gensym = Name(self.strings.len() as u32); - // leave out of `names` to avoid colliding - let string = self.strings[name.0 as usize].clone(); - self.strings.push(string); - gensym - } - - pub fn get(&self, name: Name) -> Rc<str> { - self.strings[name.0 as usize].clone() - } - - pub fn find(&self, string: &str) -> Option<Name> { - self.names.get(string).cloned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use ast::Name; - - #[test] - fn interner_tests() { - let mut i: Interner = Interner::new(); - // first one is zero: - assert_eq!(i.intern("dog"), Name(0)); - // re-use gets the same entry: - assert_eq!(i.intern ("dog"), Name(0)); - // different string gets a different #: - assert_eq!(i.intern("cat"), Name(1)); - assert_eq!(i.intern("cat"), Name(1)); - // dog is still at zero - assert_eq!(i.intern("dog"), Name(0)); - // gensym gets 3 - assert_eq!(i.gensym("zebra"), Name(2)); - // gensym of same string gets new number : - assert_eq!(i.gensym("zebra"), Name(3)); - // gensym of *existing* string gets new number: - assert_eq!(i.gensym("dog"), Name(4)); - // gensym tests again with gensym_copy: - assert_eq!(i.gensym_copy(Name(2)), Name(5)); - assert_eq!(&*i.get(Name(5)), "zebra"); - assert_eq!(i.gensym_copy(Name(2)), Name(6)); - assert_eq!(&*i.get(Name(6)), "zebra"); - assert_eq!(&*i.get(Name(0)), "dog"); - assert_eq!(&*i.get(Name(1)), "cat"); - assert_eq!(&*i.get(Name(2)), "zebra"); - assert_eq!(&*i.get(Name(3)), "zebra"); - assert_eq!(&*i.get(Name(4)), "dog"); - } -} diff --git a/src/libsyntax/util/lev_distance.rs b/src/libsyntax/util/lev_distance.rs index e0796c34e57..0d6df2cfcb6 100644 --- a/src/libsyntax/util/lev_distance.rs +++ b/src/libsyntax/util/lev_distance.rs @@ -10,7 +10,7 @@ use ast::Name; use std::cmp; -use parse::token::InternedString; +use symbol::InternedString; /// To find the Levenshtein distance between two strings pub fn lev_distance(a: &str, b: &str) -> usize { diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs index df4eb1c9ed7..ce24fe1eb61 100644 --- a/src/libsyntax/util/parser.rs +++ b/src/libsyntax/util/parser.rs @@ -7,7 +7,8 @@ // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your // option. This file may not be copied, modified, or distributed // except according to those terms. -use parse::token::{Token, BinOpToken, keywords}; +use parse::token::{Token, BinOpToken}; +use symbol::keywords; use ast::BinOpKind; /// Associative operator with precedence. |
