diff options
Diffstat (limited to 'src/libsyntax/parse/token.rs')
| -rw-r--r-- | src/libsyntax/parse/token.rs | 170 |
1 files changed, 111 insertions, 59 deletions
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 68e2f44ebb1..d6edccd33a4 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -12,12 +12,15 @@ use ast; use ast::{P, Name, Mrk}; use ast_util; use parse::token; -use util::interner::StrInterner; +use util::interner::{RcStr, StrInterner}; use util::interner; +use extra::serialize::{Decodable, Decoder, Encodable, Encoder}; use std::cast; use std::char; +use std::fmt; use std::local_data; +use std::path::BytesContainer; #[allow(non_camel_case_types)] #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)] @@ -185,32 +188,44 @@ pub fn to_str(input: @IdentInterner, t: &Token) -> ~str { } LIT_INT_UNSUFFIXED(i) => { i.to_str() } LIT_FLOAT(ref s, t) => { - let mut body = ident_to_str(s).to_owned(); + let body_string = get_ident(s.name); + let mut body = body_string.get().to_str(); if body.ends_with(".") { body.push_char('0'); // `10.f` is not a float literal } body + ast_util::float_ty_to_str(t) } LIT_FLOAT_UNSUFFIXED(ref s) => { - let mut body = ident_to_str(s).to_owned(); + let body_string = get_ident(s.name); + let mut body = body_string.get().to_owned(); if body.ends_with(".") { body.push_char('0'); // `10.f` is not a float literal } body } - LIT_STR(ref s) => { format!("\"{}\"", ident_to_str(s).escape_default()) } + LIT_STR(ref s) => { + let literal_string = get_ident(s.name); + format!("\"{}\"", literal_string.get().escape_default()) + } LIT_STR_RAW(ref s, n) => { + let literal_string = get_ident(s.name); format!("r{delim}\"{string}\"{delim}", - delim="#".repeat(n), string=ident_to_str(s)) + delim="#".repeat(n), string=literal_string.get()) } /* Name components */ - IDENT(s, _) => input.get(s.name).to_owned(), - LIFETIME(s) => format!("'{}", input.get(s.name)), + IDENT(s, _) => input.get(s.name).into_owned(), + LIFETIME(s) => { + let name = input.get(s.name); + format!("'{}", name.as_slice()) + } UNDERSCORE => ~"_", /* Other */ - DOC_COMMENT(ref s) => ident_to_str(s).to_owned(), + DOC_COMMENT(ref s) => { + let comment_string = get_ident(s.name); + comment_string.get().to_str() + } EOF => ~"<eof>", INTERPOLATED(ref nt) => { match nt { @@ -525,6 +540,93 @@ pub fn get_ident_interner() -> @IdentInterner { } } +/// Represents a string stored in the task-local interner. Because the +/// interner lives for the life of the task, this can be safely treated as an +/// immortal string, as long as it never crosses between tasks. +/// +/// FIXME(pcwalton): You must be careful about what you do in the destructors +/// of objects stored in TLS, because they may run after the interner is +/// destroyed. In particular, they must not access string contents. This can +/// be fixed in the future by just leaking all strings until task death +/// somehow. +#[deriving(Clone, Eq, IterBytes, Ord, TotalEq, TotalOrd)] +pub struct InternedString { + priv string: RcStr, +} + +impl InternedString { + #[inline] + pub fn new(string: &'static str) -> InternedString { + InternedString { + string: RcStr::new(string), + } + } + + #[inline] + fn new_from_rc_str(string: RcStr) -> InternedString { + InternedString { + string: string, + } + } + + #[inline] + pub fn get<'a>(&'a self) -> &'a str { + self.string.as_slice() + } +} + +impl BytesContainer for InternedString { + fn container_as_bytes<'a>(&'a self) -> &'a [u8] { + // FIXME(pcwalton): This is a workaround for the incorrect signature + // of `BytesContainer`, which is itself a workaround for the lack of + // DST. + unsafe { + let this = self.get(); + cast::transmute(this.container_as_bytes()) + } + } +} + +impl fmt::Default for InternedString { + fn fmt(obj: &InternedString, f: &mut fmt::Formatter) { + write!(f.buf, "{}", obj.string.as_slice()); + } +} + +impl<'a> Equiv<&'a str> for InternedString { + fn equiv(&self, other: & &'a str) -> bool { + (*other) == self.string.as_slice() + } +} + +impl<D:Decoder> Decodable<D> for InternedString { + fn decode(d: &mut D) -> InternedString { + let interner = get_ident_interner(); + get_ident(interner.intern(d.read_str())) + } +} + +impl<E:Encoder> Encodable<E> for InternedString { + fn encode(&self, e: &mut E) { + e.emit_str(self.string.as_slice()) + } +} + +/// Returns the string contents of an identifier, using the task-local +/// interner. +#[inline] +pub fn get_ident(idx: Name) -> InternedString { + let interner = get_ident_interner(); + InternedString::new_from_rc_str(interner.get(idx)) +} + +/// Interns and returns the string contents of an identifier, using the +/// task-local interner. +#[inline] +pub fn intern_and_get_ident(s: &str) -> InternedString { + get_ident(intern(s)) +} + /* for when we don't care about the contents; doesn't interact with TLD or serialization */ pub fn mk_fake_ident_interner() -> @IdentInterner { @@ -532,6 +634,7 @@ pub fn mk_fake_ident_interner() -> @IdentInterner { } // maps a string to its interned representation +#[inline] pub fn intern(str : &str) -> Name { let interner = get_ident_interner(); interner.intern(str) @@ -543,16 +646,6 @@ pub fn gensym(str : &str) -> Name { interner.gensym(str) } -// map an interned representation back to a string -pub fn interner_get(name : Name) -> @str { - get_ident_interner().get(name) -} - -// maps an identifier to the string that it corresponds to -pub fn ident_to_str(id : &ast::Ident) -> @str { - interner_get(id.name) -} - // maps a string to an identifier with an empty syntax context pub fn str_to_ident(str : &str) -> ast::Ident { ast::Ident::new(intern(str)) @@ -576,28 +669,6 @@ pub fn fresh_name(src : &ast::Ident) -> Name { gensym(format!("{}_{}",ident_to_str(src),num))*/ } -// it looks like there oughta be a str_ptr_eq fn, but no one bothered to implement it? - -// determine whether two @str values are pointer-equal -pub fn str_ptr_eq(a : @str, b : @str) -> bool { - unsafe { - let p : uint = cast::transmute(a); - let q : uint = cast::transmute(b); - let result = p == q; - // got to transmute them back, to make sure the ref count is correct: - let _junk1 : @str = cast::transmute(p); - let _junk2 : @str = cast::transmute(q); - result - } -} - -// return true when two identifiers refer (through the intern table) to the same ptr_eq -// string. This is used to compare identifiers in places where hygienic comparison is -// not wanted (i.e. not lexical vars). -pub fn ident_spelling_eq(a : &ast::Ident, b : &ast::Ident) -> bool { - str_ptr_eq(interner_get(a.name),interner_get(b.name)) -} - // create a fresh mark. pub fn fresh_mark() -> Mrk { gensym("mark") @@ -669,23 +740,4 @@ mod test { let a1 = mark_ident(a,92); assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false))); } - - - #[test] fn str_ptr_eq_tests(){ - let a = @"abc"; - let b = @"abc"; - let c = a; - assert!(str_ptr_eq(a,c)); - assert!(!str_ptr_eq(a,b)); - } - - #[test] fn fresh_name_pointer_sharing() { - let ghi = str_to_ident("ghi"); - assert_eq!(ident_to_str(&ghi),@"ghi"); - assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&ghi))) - let fresh = ast::Ident::new(fresh_name(&ghi)); - assert_eq!(ident_to_str(&fresh),@"ghi"); - assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&fresh))); - } - } |