diff options
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/attr.rs | 6 | ||||
| -rw-r--r-- | src/libsyntax/parse/comments.rs | 5 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer.rs | 41 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 120 | ||||
| -rw-r--r-- | src/libsyntax/parse/obsolete.rs | 14 | ||||
| -rw-r--r-- | src/libsyntax/parse/parser.rs | 113 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 170 |
7 files changed, 258 insertions, 211 deletions
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index e7630a66855..c9bea78d02d 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -45,7 +45,7 @@ impl ParserAttr for Parser { } token::DOC_COMMENT(s) => { let attr = ::attr::mk_sugared_doc_attr( - self.id_to_str(s), + self.id_to_interned_str(s), self.span.lo, self.span.hi ); @@ -133,7 +133,7 @@ impl ParserAttr for Parser { } token::DOC_COMMENT(s) => { self.bump(); - ::attr::mk_sugared_doc_attr(self.id_to_str(s), + ::attr::mk_sugared_doc_attr(self.id_to_interned_str(s), self.span.lo, self.span.hi) } @@ -157,7 +157,7 @@ impl ParserAttr for Parser { fn parse_meta_item(&mut self) -> @ast::MetaItem { let lo = self.span.lo; let ident = self.parse_ident(); - let name = self.id_to_str(ident); + let name = self.id_to_interned_str(ident); match self.token { token::EQ => { self.bump(); diff --git a/src/libsyntax/parse/comments.rs b/src/libsyntax/parse/comments.rs index aa5e4e01ae0..7165e7b404f 100644 --- a/src/libsyntax/parse/comments.rs +++ b/src/libsyntax/parse/comments.rs @@ -54,7 +54,6 @@ pub fn doc_comment_style(comment: &str) -> ast::AttrStyle { } pub fn strip_doc_comment_decoration(comment: &str) -> ~str { - /// remove whitespace-only lines from the start/end of lines fn vertical_trim(lines: ~[~str]) -> ~[~str] { let mut i = 0u; @@ -348,10 +347,10 @@ pub struct Literal { // probably not a good thing. pub fn gather_comments_and_literals(span_diagnostic: @diagnostic::SpanHandler, - path: @str, + path: ~str, srdr: &mut io::Reader) -> (~[Comment], ~[Literal]) { - let src = str::from_utf8_owned(srdr.read_to_end()).unwrap().to_managed(); + let src = str::from_utf8_owned(srdr.read_to_end()).unwrap(); let cm = CodeMap::new(); let filemap = cm.new_filemap(path, src); let rdr = lexer::new_low_level_string_reader(span_diagnostic, filemap); diff --git a/src/libsyntax/parse/lexer.rs b/src/libsyntax/parse/lexer.rs index 2521bb515f7..8c55990289a 100644 --- a/src/libsyntax/parse/lexer.rs +++ b/src/libsyntax/parse/lexer.rs @@ -42,7 +42,6 @@ pub struct TokenAndSpan { pub struct StringReader { span_diagnostic: @SpanHandler, - src: @str, // The absolute offset within the codemap of the next character to read pos: Cell<BytePos>, // The absolute offset within the codemap of the last character read(curr) @@ -73,7 +72,6 @@ pub fn new_low_level_string_reader(span_diagnostic: @SpanHandler, let initial_char = '\n'; let r = @StringReader { span_diagnostic: span_diagnostic, - src: filemap.src, pos: Cell::new(filemap.start_pos), last_pos: Cell::new(filemap.start_pos), col: Cell::new(CharPos(0)), @@ -93,7 +91,6 @@ pub fn new_low_level_string_reader(span_diagnostic: @SpanHandler, fn dup_string_reader(r: @StringReader) -> @StringReader { @StringReader { span_diagnostic: r.span_diagnostic, - src: r.src, pos: Cell::new(r.pos.get()), last_pos: Cell::new(r.last_pos.get()), col: Cell::new(r.col.get()), @@ -188,7 +185,7 @@ fn fatal_span_verbose(rdr: @StringReader, -> ! { let mut m = m; m.push_str(": "); - let s = rdr.src.slice( + let s = rdr.filemap.src.slice( byte_offset(rdr, from_pos).to_uint(), byte_offset(rdr, to_pos).to_uint()); m.push_str(s); @@ -239,7 +236,7 @@ fn with_str_from_to<T>( end: BytePos, f: |s: &str| -> T) -> T { - f(rdr.src.slice( + f(rdr.filemap.src.slice( byte_offset(rdr, start).to_uint(), byte_offset(rdr, end).to_uint())) } @@ -249,12 +246,12 @@ fn with_str_from_to<T>( pub fn bump(rdr: &StringReader) { rdr.last_pos.set(rdr.pos.get()); let current_byte_offset = byte_offset(rdr, rdr.pos.get()).to_uint(); - if current_byte_offset < (rdr.src).len() { + if current_byte_offset < (rdr.filemap.src).len() { assert!(rdr.curr.get() != unsafe { transmute(-1u32) }); // FIXME: #8971: unsound let last_char = rdr.curr.get(); - let next = rdr.src.char_range_at(current_byte_offset); + let next = rdr.filemap.src.char_range_at(current_byte_offset); let byte_offset_diff = next.next - current_byte_offset; rdr.pos.set(rdr.pos.get() + Pos::from_uint(byte_offset_diff)); rdr.curr.set(next.ch); @@ -277,8 +274,8 @@ pub fn is_eof(rdr: @StringReader) -> bool { } pub fn nextch(rdr: @StringReader) -> char { let offset = byte_offset(rdr, rdr.pos.get()).to_uint(); - if offset < (rdr.src).len() { - return rdr.src.char_at(offset); + if offset < (rdr.filemap.src).len() { + return rdr.filemap.src.char_at(offset); } else { return unsafe { transmute(-1u32) }; } // FIXME: #8971: unsound } @@ -975,9 +972,9 @@ mod test { } // open a string reader for the given string - fn setup(teststr: @str) -> Env { + fn setup(teststr: ~str) -> Env { let cm = CodeMap::new(); - let fm = cm.new_filemap(@"zebra.rs", teststr); + let fm = cm.new_filemap(~"zebra.rs", teststr); let span_handler = diagnostic::mk_span_handler(diagnostic::mk_handler(None),@cm); Env { @@ -987,7 +984,7 @@ mod test { #[test] fn t1 () { let Env {string_reader} = - setup(@"/* my source file */ \ + setup(~"/* my source file */ \ fn main() { println!(\"zebra\"); }\n"); let id = str_to_ident("fn"); let tok1 = string_reader.next_token(); @@ -1023,14 +1020,14 @@ mod test { } #[test] fn doublecolonparsing () { - let env = setup (@"a b"); + let env = setup (~"a b"); check_tokenization (env, ~[mk_ident("a",false), mk_ident("b",false)]); } #[test] fn dcparsing_2 () { - let env = setup (@"a::b"); + let env = setup (~"a::b"); check_tokenization (env, ~[mk_ident("a",true), token::MOD_SEP, @@ -1038,7 +1035,7 @@ mod test { } #[test] fn dcparsing_3 () { - let env = setup (@"a ::b"); + let env = setup (~"a ::b"); check_tokenization (env, ~[mk_ident("a",false), token::MOD_SEP, @@ -1046,7 +1043,7 @@ mod test { } #[test] fn dcparsing_4 () { - let env = setup (@"a:: b"); + let env = setup (~"a:: b"); check_tokenization (env, ~[mk_ident("a",true), token::MOD_SEP, @@ -1054,28 +1051,28 @@ mod test { } #[test] fn character_a() { - let env = setup(@"'a'"); + let env = setup(~"'a'"); let TokenAndSpan {tok, sp: _} = env.string_reader.next_token(); assert_eq!(tok,token::LIT_CHAR('a' as u32)); } #[test] fn character_space() { - let env = setup(@"' '"); + let env = setup(~"' '"); let TokenAndSpan {tok, sp: _} = env.string_reader.next_token(); assert_eq!(tok, token::LIT_CHAR(' ' as u32)); } #[test] fn character_escaped() { - let env = setup(@"'\\n'"); + let env = setup(~"'\\n'"); let TokenAndSpan {tok, sp: _} = env.string_reader.next_token(); assert_eq!(tok, token::LIT_CHAR('\n' as u32)); } #[test] fn lifetime_name() { - let env = setup(@"'abc"); + let env = setup(~"'abc"); let TokenAndSpan {tok, sp: _} = env.string_reader.next_token(); let id = token::str_to_ident("abc"); @@ -1083,7 +1080,7 @@ mod test { } #[test] fn raw_string() { - let env = setup(@"r###\"\"#a\\b\x00c\"\"###"); + let env = setup(~"r###\"\"#a\\b\x00c\"\"###"); let TokenAndSpan {tok, sp: _} = env.string_reader.next_token(); let id = token::str_to_ident("\"#a\\b\x00c\""); @@ -1097,7 +1094,7 @@ mod test { } #[test] fn nested_block_comments() { - let env = setup(@"/* /* */ */'a'"); + let env = setup(~"/* /* */ */'a'"); let TokenAndSpan {tok, sp: _} = env.string_reader.next_token(); assert_eq!(tok,token::LIT_CHAR('a' as u32)); diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index e026a11cafe..cec9f7c2d9f 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -89,12 +89,11 @@ pub fn parse_crate_attrs_from_file( return inner; } -pub fn parse_crate_from_source_str( - name: @str, - source: @str, - cfg: ast::CrateConfig, - sess: @ParseSess -) -> ast::Crate { +pub fn parse_crate_from_source_str(name: ~str, + source: ~str, + cfg: ast::CrateConfig, + sess: @ParseSess) + -> ast::Crate { let mut p = new_parser_from_source_str(sess, /*bad*/ cfg.clone(), name, @@ -102,12 +101,11 @@ pub fn parse_crate_from_source_str( maybe_aborted(p.parse_crate_mod(),p) } -pub fn parse_crate_attrs_from_source_str( - name: @str, - source: @str, - cfg: ast::CrateConfig, - sess: @ParseSess -) -> ~[ast::Attribute] { +pub fn parse_crate_attrs_from_source_str(name: ~str, + source: ~str, + cfg: ast::CrateConfig, + sess: @ParseSess) + -> ~[ast::Attribute] { let mut p = new_parser_from_source_str(sess, /*bad*/ cfg.clone(), name, @@ -116,44 +114,40 @@ pub fn parse_crate_attrs_from_source_str( return inner; } -pub fn parse_expr_from_source_str( - name: @str, - source: @str, - cfg: ast::CrateConfig, - sess: @ParseSess -) -> @ast::Expr { +pub fn parse_expr_from_source_str(name: ~str, + source: ~str, + cfg: ast::CrateConfig, + sess: @ParseSess) + -> @ast::Expr { let mut p = new_parser_from_source_str(sess, cfg, name, source); maybe_aborted(p.parse_expr(), p) } -pub fn parse_item_from_source_str( - name: @str, - source: @str, - cfg: ast::CrateConfig, - sess: @ParseSess -) -> Option<@ast::Item> { +pub fn parse_item_from_source_str(name: ~str, + source: ~str, + cfg: ast::CrateConfig, + sess: @ParseSess) + -> Option<@ast::Item> { let mut p = new_parser_from_source_str(sess, cfg, name, source); let attrs = p.parse_outer_attributes(); maybe_aborted(p.parse_item(attrs),p) } -pub fn parse_meta_from_source_str( - name: @str, - source: @str, - cfg: ast::CrateConfig, - sess: @ParseSess -) -> @ast::MetaItem { +pub fn parse_meta_from_source_str(name: ~str, + source: ~str, + cfg: ast::CrateConfig, + sess: @ParseSess) + -> @ast::MetaItem { let mut p = new_parser_from_source_str(sess, cfg, name, source); maybe_aborted(p.parse_meta_item(),p) } -pub fn parse_stmt_from_source_str( - name: @str, - source: @str, - cfg: ast::CrateConfig, - attrs: ~[ast::Attribute], - sess: @ParseSess -) -> @ast::Stmt { +pub fn parse_stmt_from_source_str(name: ~str, + source: ~str, + cfg: ast::CrateConfig, + attrs: ~[ast::Attribute], + sess: @ParseSess) + -> @ast::Stmt { let mut p = new_parser_from_source_str( sess, cfg, @@ -163,12 +157,11 @@ pub fn parse_stmt_from_source_str( maybe_aborted(p.parse_stmt(attrs),p) } -pub fn parse_tts_from_source_str( - name: @str, - source: @str, - cfg: ast::CrateConfig, - sess: @ParseSess -) -> ~[ast::TokenTree] { +pub fn parse_tts_from_source_str(name: ~str, + source: ~str, + cfg: ast::CrateConfig, + sess: @ParseSess) + -> ~[ast::TokenTree] { let mut p = new_parser_from_source_str( sess, cfg, @@ -183,9 +176,9 @@ pub fn parse_tts_from_source_str( // Create a new parser from a source string pub fn new_parser_from_source_str(sess: @ParseSess, cfg: ast::CrateConfig, - name: @str, - source: @str) - -> Parser { + name: ~str, + source: ~str) + -> Parser { filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg) } @@ -248,20 +241,17 @@ pub fn file_to_filemap(sess: @ParseSess, path: &Path, spanopt: Option<Span>) }; match str::from_utf8_owned(bytes) { Some(s) => { - return string_to_filemap(sess, s.to_managed(), - path.as_str().unwrap().to_managed()); - } - None => { - err(format!("{} is not UTF-8 encoded", path.display())) + return string_to_filemap(sess, s, path.as_str().unwrap().to_str()) } + None => err(format!("{} is not UTF-8 encoded", path.display())), } unreachable!() } // given a session and a string, add the string to // the session's codemap and return the new filemap -pub fn string_to_filemap(sess: @ParseSess, source: @str, path: @str) - -> @FileMap { +pub fn string_to_filemap(sess: @ParseSess, source: ~str, path: ~str) + -> @FileMap { sess.cm.new_filemap(path, source) } @@ -324,7 +314,7 @@ mod test { } #[test] fn path_exprs_1() { - assert_eq!(string_to_expr(@"a"), + assert_eq!(string_to_expr(~"a"), @ast::Expr{ id: ast::DUMMY_NODE_ID, node: ast::ExprPath(ast::Path { @@ -343,7 +333,7 @@ mod test { } #[test] fn path_exprs_2 () { - assert_eq!(string_to_expr(@"::a::b"), + assert_eq!(string_to_expr(~"::a::b"), @ast::Expr { id: ast::DUMMY_NODE_ID, node: ast::ExprPath(ast::Path { @@ -368,12 +358,12 @@ mod test { #[should_fail] #[test] fn bad_path_expr_1() { - string_to_expr(@"::abc::def::return"); + string_to_expr(~"::abc::def::return"); } // check the token-tree-ization of macros #[test] fn string_to_tts_macro () { - let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))"); + let tts = string_to_tts(~"macro_rules! zip (($a)=>($a))"); match tts { [ast::TTTok(_,_), ast::TTTok(_,token::NOT), @@ -417,7 +407,7 @@ mod test { } #[test] fn string_to_tts_1 () { - let tts = string_to_tts(@"fn a (b : int) { b; }"); + let tts = string_to_tts(~"fn a (b : int) { b; }"); assert_eq!(to_json_str(&tts), ~"[\ {\ @@ -546,7 +536,7 @@ mod test { } #[test] fn ret_expr() { - assert_eq!(string_to_expr(@"return d"), + assert_eq!(string_to_expr(~"return d"), @ast::Expr{ id: ast::DUMMY_NODE_ID, node:ast::ExprRet(Some(@ast::Expr{ @@ -569,7 +559,7 @@ mod test { } #[test] fn parse_stmt_1 () { - assert_eq!(string_to_stmt(@"b;"), + assert_eq!(string_to_stmt(~"b;"), @Spanned{ node: ast::StmtExpr(@ast::Expr { id: ast::DUMMY_NODE_ID, @@ -595,7 +585,7 @@ mod test { } #[test] fn parse_ident_pat () { - let mut parser = string_to_parser(@"b"); + let mut parser = string_to_parser(~"b"); assert_eq!(parser.parse_pat(), @ast::Pat{id: ast::DUMMY_NODE_ID, node: ast::PatIdent( @@ -619,7 +609,7 @@ mod test { // check the contents of the tt manually: #[test] fn parse_fundecl () { // this test depends on the intern order of "fn" and "int" - assert_eq!(string_to_item(@"fn a (b : int) { b; }"), + assert_eq!(string_to_item(~"fn a (b : int) { b; }"), Some( @ast::Item{ident:str_to_ident("a"), attrs:~[], @@ -711,12 +701,12 @@ mod test { #[test] fn parse_exprs () { // just make sure that they parse.... - string_to_expr(@"3 + 4"); - string_to_expr(@"a::z.froob(b,@(987+3))"); + string_to_expr(~"3 + 4"); + string_to_expr(~"a::z.froob(b,@(987+3))"); } #[test] fn attrs_fix_bug () { - string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) + string_to_item(~"pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) -> Result<@Writer, ~str> { #[cfg(windows)] fn wb() -> c_int { diff --git a/src/libsyntax/parse/obsolete.rs b/src/libsyntax/parse/obsolete.rs index c4887d55e2a..b85d89cf804 100644 --- a/src/libsyntax/parse/obsolete.rs +++ b/src/libsyntax/parse/obsolete.rs @@ -22,7 +22,6 @@ use codemap::{Span, respan}; use parse::parser::Parser; use parse::token; -use std::str; use std::to_bytes; /// The specific types of unsupported syntax @@ -45,6 +44,8 @@ pub enum ObsoleteSyntax { ObsoleteMultipleImport, ObsoleteExternModAttributesInParens, ObsoleteManagedPattern, + ObsoleteManagedString, + ObsoleteManagedVec, } impl to_bytes::IterBytes for ObsoleteSyntax { @@ -150,6 +151,14 @@ impl ParserObsoleteMethods for Parser { "use a nested `match` expression instead of a managed box \ pattern" ), + ObsoleteManagedString => ( + "managed string", + "use `Rc<~str>` instead of a managed string" + ), + ObsoleteManagedVec => ( + "managed vector", + "use `Rc<~[T]>` instead of a managed vector" + ), }; self.report(sp, kind, kind_str, desc); @@ -178,7 +187,8 @@ impl ParserObsoleteMethods for Parser { fn is_obsolete_ident(&mut self, ident: &str) -> bool { match self.token { token::IDENT(sid, _) => { - str::eq_slice(self.id_to_str(sid), ident) + let interned_string = token::get_ident(sid.name); + interned_string.equiv(&ident) } _ => false } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 642624adfb2..dd7cc3a2314 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -29,7 +29,7 @@ use ast::{ExprField, ExprFnBlock, ExprIf, ExprIndex}; use ast::{ExprLit, ExprLogLevel, ExprLoop, ExprMac}; use ast::{ExprMethodCall, ExprParen, ExprPath, ExprProc}; use ast::{ExprRepeat, ExprRet, ExprStruct, ExprTup, ExprUnary}; -use ast::{ExprVec, ExprVstore, ExprVstoreSlice, ExprVstoreBox}; +use ast::{ExprVec, ExprVstore, ExprVstoreSlice}; use ast::{ExprVstoreMutSlice, ExprWhile, ExprForLoop, ExternFn, Field, FnDecl}; use ast::{ExprVstoreUniq, Onceness, Once, Many}; use ast::{ForeignItem, ForeignItemStatic, ForeignItemFn, ForeignMod}; @@ -71,10 +71,9 @@ use parse::common::{seq_sep_trailing_disallowed, seq_sep_trailing_allowed}; use parse::lexer::Reader; use parse::lexer::TokenAndSpan; use parse::obsolete::*; -use parse::token::{can_begin_expr, get_ident_interner, ident_to_str, is_ident}; -use parse::token::{is_ident_or_path}; -use parse::token::{is_plain_ident, INTERPOLATED, keywords, special_idents}; -use parse::token::{token_to_binop}; +use parse::token::{INTERPOLATED, InternedString, can_begin_expr, get_ident}; +use parse::token::{get_ident_interner, is_ident, is_ident_or_path}; +use parse::token::{is_plain_ident, keywords, special_idents, token_to_binop}; use parse::token; use parse::{new_sub_parser_from_file, ParseSess}; use opt_vec; @@ -345,7 +344,7 @@ pub struct Parser { /// extra detail when the same error is seen twice obsolete_set: HashSet<ObsoleteSyntax>, /// Used to determine the path to externally loaded source files - mod_path_stack: ~[@str], + mod_path_stack: ~[InternedString], /// Stack of spans of open delimiters. Used for error message. open_braces: ~[Span], /* do not copy the parser; its state is tied to outside state */ @@ -531,10 +530,11 @@ impl Parser { // otherwise, eat it. pub fn expect_keyword(&mut self, kw: keywords::Keyword) { if !self.eat_keyword(kw) { - let id_str = self.id_to_str(kw.to_ident()).to_str(); + let id_ident = kw.to_ident(); + let id_interned_str = token::get_ident(id_ident.name); let token_str = self.this_token_to_str(); self.fatal(format!("expected `{}`, found `{}`", - id_str, + id_interned_str.get(), token_str)) } } @@ -802,8 +802,8 @@ impl Parser { self.sess.span_diagnostic.handler().abort_if_errors(); } - pub fn id_to_str(&mut self, id: Ident) -> @str { - get_ident_interner().get(id.name) + pub fn id_to_interned_str(&mut self, id: Ident) -> InternedString { + get_ident(id.name) } // Is the current token one of the keywords that signals a bare function @@ -1291,7 +1291,7 @@ impl Parser { } // other things are parsed as @/~ + a type. Note that constructs like - // @[] and @str will be resolved during typeck to slices and so forth, + // ~[] and ~str will be resolved during typeck to slices and so forth, // rather than boxed ptrs. But the special casing of str/vec is not // reflected in the AST type. if sigil == OwnedSigil { @@ -1401,11 +1401,18 @@ impl Parser { token::LIT_INT(i, it) => LitInt(i, it), token::LIT_UINT(u, ut) => LitUint(u, ut), token::LIT_INT_UNSUFFIXED(i) => LitIntUnsuffixed(i), - token::LIT_FLOAT(s, ft) => LitFloat(self.id_to_str(s), ft), - token::LIT_FLOAT_UNSUFFIXED(s) => - LitFloatUnsuffixed(self.id_to_str(s)), - token::LIT_STR(s) => LitStr(self.id_to_str(s), ast::CookedStr), - token::LIT_STR_RAW(s, n) => LitStr(self.id_to_str(s), ast::RawStr(n)), + token::LIT_FLOAT(s, ft) => { + LitFloat(self.id_to_interned_str(s), ft) + } + token::LIT_FLOAT_UNSUFFIXED(s) => { + LitFloatUnsuffixed(self.id_to_interned_str(s)) + } + token::LIT_STR(s) => { + LitStr(self.id_to_interned_str(s), ast::CookedStr) + } + token::LIT_STR_RAW(s, n) => { + LitStr(self.id_to_interned_str(s), ast::RawStr(n)) + } token::LPAREN => { self.expect(&token::RPAREN); LitNil }, _ => { self.unexpected_last(tok); } } @@ -2284,11 +2291,19 @@ impl Parser { self.bump(); let e = self.parse_prefix_expr(); hi = e.span.hi; - // HACK: turn @[...] into a @-vec + // HACK: pretending @[] is a (removed) @-vec ex = match e.node { ExprVec(..) | - ExprRepeat(..) => ExprVstore(e, ExprVstoreBox), - ExprLit(lit) if lit_is_str(lit) => ExprVstore(e, ExprVstoreBox), + ExprRepeat(..) => { + self.obsolete(e.span, ObsoleteManagedVec); + // the above error means that no-one will know we're + // lying... hopefully. + ExprVstore(e, ExprVstoreUniq) + } + ExprLit(lit) if lit_is_str(lit) => { + self.obsolete(self.last_span, ObsoleteManagedString); + ExprVstore(e, ExprVstoreUniq) + } _ => self.mk_unary(UnBox, e) }; } @@ -2806,34 +2821,11 @@ impl Parser { token::AT => { self.bump(); let sub = self.parse_pat(); - hi = sub.span.hi; - // HACK: parse @"..." as a literal of a vstore @str - pat = match sub.node { - PatLit(e) => { - match e.node { - ExprLit(lit) if lit_is_str(lit) => { - let vst = @Expr { - id: ast::DUMMY_NODE_ID, - node: ExprVstore(e, ExprVstoreBox), - span: mk_sp(lo, hi), - }; - PatLit(vst) - } - _ => { - self.obsolete(self.span, ObsoleteManagedPattern); - PatUniq(sub) - } - } - } - _ => { - self.obsolete(self.span, ObsoleteManagedPattern); - PatUniq(sub) - } - }; - hi = self.last_span.hi; + self.obsolete(self.span, ObsoleteManagedPattern); + let hi = self.last_span.hi; return @ast::Pat { id: ast::DUMMY_NODE_ID, - node: pat, + node: PatUniq(sub), span: mk_sp(lo, hi) } } @@ -3429,7 +3421,9 @@ impl Parser { loop { match self.token { token::LIFETIME(lifetime) => { - if "static" == self.id_to_str(lifetime) { + let lifetime_interned_string = + token::get_ident(lifetime.name); + if lifetime_interned_string.equiv(&("static")) { result.push(RegionTyParamBound); } else { self.span_err(self.span, @@ -3970,8 +3964,9 @@ impl Parser { fields.push(self.parse_struct_decl_field()); } if fields.len() == 0 { + let string = get_ident_interner().get(class_name.name); self.fatal(format!("Unit-like struct definition should be written as `struct {};`", - get_ident_interner().get(class_name.name))); + string.as_slice())); } self.bump(); } else if self.token == token::LPAREN { @@ -4142,11 +4137,11 @@ impl Parser { } fn push_mod_path(&mut self, id: Ident, attrs: &[Attribute]) { - let default_path = token::interner_get(id.name); + let default_path = self.id_to_interned_str(id); let file_path = match ::attr::first_attr_value_str_by_name(attrs, "path") { Some(d) => d, - None => default_path + None => default_path, }; self.mod_path_stack.push(file_path) } @@ -4169,7 +4164,8 @@ impl Parser { outer_attrs, "path") { Some(d) => dir_path.join(d), None => { - let mod_name = token::interner_get(id.name).to_owned(); + let mod_string = token::get_ident(id.name); + let mod_name = mod_string.get().to_owned(); let default_path_str = mod_name + ".rs"; let secondary_path_str = mod_name + "/mod.rs"; let default_path = dir_path.join(default_path_str.as_slice()); @@ -4524,7 +4520,8 @@ impl Parser { token::LIT_STR(s) | token::LIT_STR_RAW(s, _) => { self.bump(); - let the_string = ident_to_str(&s); + let identifier_string = token::get_ident(s.name); + let the_string = identifier_string.get(); let mut abis = AbiSet::empty(); for word in the_string.words() { match abi::lookup(word) { @@ -4860,7 +4857,6 @@ impl Parser { let first_ident = self.parse_ident(); let mut path = ~[first_ident]; - debug!("parsed view path: {}", self.id_to_str(first_ident)); match self.token { token::EQ => { // x = foo::bar @@ -5119,17 +5115,20 @@ impl Parser { } } - pub fn parse_optional_str(&mut self) -> Option<(@str, ast::StrStyle)> { + pub fn parse_optional_str(&mut self) + -> Option<(InternedString, ast::StrStyle)> { let (s, style) = match self.token { - token::LIT_STR(s) => (s, ast::CookedStr), - token::LIT_STR_RAW(s, n) => (s, ast::RawStr(n)), + token::LIT_STR(s) => (self.id_to_interned_str(s), ast::CookedStr), + token::LIT_STR_RAW(s, n) => { + (self.id_to_interned_str(s), ast::RawStr(n)) + } _ => return None }; self.bump(); - Some((ident_to_str(&s), style)) + Some((s, style)) } - pub fn parse_str(&mut self) -> (@str, StrStyle) { + pub fn parse_str(&mut self) -> (InternedString, StrStyle) { match self.parse_optional_str() { Some(s) => { s } _ => self.fatal("expected string literal") diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 68e2f44ebb1..d6edccd33a4 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -12,12 +12,15 @@ use ast; use ast::{P, Name, Mrk}; use ast_util; use parse::token; -use util::interner::StrInterner; +use util::interner::{RcStr, StrInterner}; use util::interner; +use extra::serialize::{Decodable, Decoder, Encodable, Encoder}; use std::cast; use std::char; +use std::fmt; use std::local_data; +use std::path::BytesContainer; #[allow(non_camel_case_types)] #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)] @@ -185,32 +188,44 @@ pub fn to_str(input: @IdentInterner, t: &Token) -> ~str { } LIT_INT_UNSUFFIXED(i) => { i.to_str() } LIT_FLOAT(ref s, t) => { - let mut body = ident_to_str(s).to_owned(); + let body_string = get_ident(s.name); + let mut body = body_string.get().to_str(); if body.ends_with(".") { body.push_char('0'); // `10.f` is not a float literal } body + ast_util::float_ty_to_str(t) } LIT_FLOAT_UNSUFFIXED(ref s) => { - let mut body = ident_to_str(s).to_owned(); + let body_string = get_ident(s.name); + let mut body = body_string.get().to_owned(); if body.ends_with(".") { body.push_char('0'); // `10.f` is not a float literal } body } - LIT_STR(ref s) => { format!("\"{}\"", ident_to_str(s).escape_default()) } + LIT_STR(ref s) => { + let literal_string = get_ident(s.name); + format!("\"{}\"", literal_string.get().escape_default()) + } LIT_STR_RAW(ref s, n) => { + let literal_string = get_ident(s.name); format!("r{delim}\"{string}\"{delim}", - delim="#".repeat(n), string=ident_to_str(s)) + delim="#".repeat(n), string=literal_string.get()) } /* Name components */ - IDENT(s, _) => input.get(s.name).to_owned(), - LIFETIME(s) => format!("'{}", input.get(s.name)), + IDENT(s, _) => input.get(s.name).into_owned(), + LIFETIME(s) => { + let name = input.get(s.name); + format!("'{}", name.as_slice()) + } UNDERSCORE => ~"_", /* Other */ - DOC_COMMENT(ref s) => ident_to_str(s).to_owned(), + DOC_COMMENT(ref s) => { + let comment_string = get_ident(s.name); + comment_string.get().to_str() + } EOF => ~"<eof>", INTERPOLATED(ref nt) => { match nt { @@ -525,6 +540,93 @@ pub fn get_ident_interner() -> @IdentInterner { } } +/// Represents a string stored in the task-local interner. Because the +/// interner lives for the life of the task, this can be safely treated as an +/// immortal string, as long as it never crosses between tasks. +/// +/// FIXME(pcwalton): You must be careful about what you do in the destructors +/// of objects stored in TLS, because they may run after the interner is +/// destroyed. In particular, they must not access string contents. This can +/// be fixed in the future by just leaking all strings until task death +/// somehow. +#[deriving(Clone, Eq, IterBytes, Ord, TotalEq, TotalOrd)] +pub struct InternedString { + priv string: RcStr, +} + +impl InternedString { + #[inline] + pub fn new(string: &'static str) -> InternedString { + InternedString { + string: RcStr::new(string), + } + } + + #[inline] + fn new_from_rc_str(string: RcStr) -> InternedString { + InternedString { + string: string, + } + } + + #[inline] + pub fn get<'a>(&'a self) -> &'a str { + self.string.as_slice() + } +} + +impl BytesContainer for InternedString { + fn container_as_bytes<'a>(&'a self) -> &'a [u8] { + // FIXME(pcwalton): This is a workaround for the incorrect signature + // of `BytesContainer`, which is itself a workaround for the lack of + // DST. + unsafe { + let this = self.get(); + cast::transmute(this.container_as_bytes()) + } + } +} + +impl fmt::Default for InternedString { + fn fmt(obj: &InternedString, f: &mut fmt::Formatter) { + write!(f.buf, "{}", obj.string.as_slice()); + } +} + +impl<'a> Equiv<&'a str> for InternedString { + fn equiv(&self, other: & &'a str) -> bool { + (*other) == self.string.as_slice() + } +} + +impl<D:Decoder> Decodable<D> for InternedString { + fn decode(d: &mut D) -> InternedString { + let interner = get_ident_interner(); + get_ident(interner.intern(d.read_str())) + } +} + +impl<E:Encoder> Encodable<E> for InternedString { + fn encode(&self, e: &mut E) { + e.emit_str(self.string.as_slice()) + } +} + +/// Returns the string contents of an identifier, using the task-local +/// interner. +#[inline] +pub fn get_ident(idx: Name) -> InternedString { + let interner = get_ident_interner(); + InternedString::new_from_rc_str(interner.get(idx)) +} + +/// Interns and returns the string contents of an identifier, using the +/// task-local interner. +#[inline] +pub fn intern_and_get_ident(s: &str) -> InternedString { + get_ident(intern(s)) +} + /* for when we don't care about the contents; doesn't interact with TLD or serialization */ pub fn mk_fake_ident_interner() -> @IdentInterner { @@ -532,6 +634,7 @@ pub fn mk_fake_ident_interner() -> @IdentInterner { } // maps a string to its interned representation +#[inline] pub fn intern(str : &str) -> Name { let interner = get_ident_interner(); interner.intern(str) @@ -543,16 +646,6 @@ pub fn gensym(str : &str) -> Name { interner.gensym(str) } -// map an interned representation back to a string -pub fn interner_get(name : Name) -> @str { - get_ident_interner().get(name) -} - -// maps an identifier to the string that it corresponds to -pub fn ident_to_str(id : &ast::Ident) -> @str { - interner_get(id.name) -} - // maps a string to an identifier with an empty syntax context pub fn str_to_ident(str : &str) -> ast::Ident { ast::Ident::new(intern(str)) @@ -576,28 +669,6 @@ pub fn fresh_name(src : &ast::Ident) -> Name { gensym(format!("{}_{}",ident_to_str(src),num))*/ } -// it looks like there oughta be a str_ptr_eq fn, but no one bothered to implement it? - -// determine whether two @str values are pointer-equal -pub fn str_ptr_eq(a : @str, b : @str) -> bool { - unsafe { - let p : uint = cast::transmute(a); - let q : uint = cast::transmute(b); - let result = p == q; - // got to transmute them back, to make sure the ref count is correct: - let _junk1 : @str = cast::transmute(p); - let _junk2 : @str = cast::transmute(q); - result - } -} - -// return true when two identifiers refer (through the intern table) to the same ptr_eq -// string. This is used to compare identifiers in places where hygienic comparison is -// not wanted (i.e. not lexical vars). -pub fn ident_spelling_eq(a : &ast::Ident, b : &ast::Ident) -> bool { - str_ptr_eq(interner_get(a.name),interner_get(b.name)) -} - // create a fresh mark. pub fn fresh_mark() -> Mrk { gensym("mark") @@ -669,23 +740,4 @@ mod test { let a1 = mark_ident(a,92); assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false))); } - - - #[test] fn str_ptr_eq_tests(){ - let a = @"abc"; - let b = @"abc"; - let c = a; - assert!(str_ptr_eq(a,c)); - assert!(!str_ptr_eq(a,b)); - } - - #[test] fn fresh_name_pointer_sharing() { - let ghi = str_to_ident("ghi"); - assert_eq!(ident_to_str(&ghi),@"ghi"); - assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&ghi))) - let fresh = ast::Ident::new(fresh_name(&ghi)); - assert_eq!(ident_to_str(&fresh),@"ghi"); - assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&fresh))); - } - } |