diff options
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 33 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 154 |
2 files changed, 65 insertions, 122 deletions
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index bbe1ddfd4cf..8d3e93d35dd 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -76,6 +76,10 @@ pub struct StringReader<'a> { // are revised to go directly to token-trees. /// Is \x00<name>,<ctxt>\x00 is interpreted as encoded ast::Ident? read_embedded_ident: bool, + + // cache a direct reference to the source text, so that we don't have to + // retrieve it via `self.filemap.src.as_ref().unwrap()` all the time. + source_text: Rc<String> } impl<'a> Reader for StringReader<'a> { @@ -141,7 +145,14 @@ pub fn make_reader_with_embedded_idents<'b>(span_diagnostic: &'b SpanHandler, impl<'a> StringReader<'a> { /// For comments.rs, which hackily pokes into pos and curr pub fn new_raw<'b>(span_diagnostic: &'b SpanHandler, - filemap: Rc<codemap::FileMap>) -> StringReader<'b> { + filemap: Rc<codemap::FileMap>) -> StringReader<'b> { + if filemap.src.is_none() { + span_diagnostic.handler.bug(&format!("Cannot lex filemap without source: {}", + filemap.name)[..]); + } + + let source_text = (*filemap.src.as_ref().unwrap()).clone(); + let mut sr = StringReader { span_diagnostic: span_diagnostic, pos: filemap.start_pos, @@ -153,6 +164,7 @@ impl<'a> StringReader<'a> { peek_tok: token::Eof, peek_span: codemap::DUMMY_SP, read_embedded_ident: false, + source_text: source_text }; sr.bump(); sr @@ -213,7 +225,7 @@ impl<'a> StringReader<'a> { m.push_str(": "); let from = self.byte_offset(from_pos).to_usize(); let to = self.byte_offset(to_pos).to_usize(); - m.push_str(&self.filemap.src[from..to]); + m.push_str(&self.source_text[from..to]); self.fatal_span_(from_pos, to_pos, &m[..]); } @@ -270,9 +282,8 @@ impl<'a> StringReader<'a> { fn with_str_from_to<T, F>(&self, start: BytePos, end: BytePos, f: F) -> T where F: FnOnce(&str) -> T, { - f(&self.filemap.src[ - self.byte_offset(start).to_usize().. - self.byte_offset(end).to_usize()]) + f(&self.source_text[self.byte_offset(start).to_usize().. + self.byte_offset(end).to_usize()]) } /// Converts CRLF to LF in the given string, raising an error on bare CR. @@ -321,12 +332,10 @@ impl<'a> StringReader<'a> { pub fn bump(&mut self) { self.last_pos = self.pos; let current_byte_offset = self.byte_offset(self.pos).to_usize(); - if current_byte_offset < self.filemap.src.len() { + if current_byte_offset < self.source_text.len() { assert!(self.curr.is_some()); let last_char = self.curr.unwrap(); - let next = self.filemap - .src - .char_range_at(current_byte_offset); + let next = self.source_text.char_range_at(current_byte_offset); let byte_offset_diff = next.next - current_byte_offset; self.pos = self.pos + Pos::from_usize(byte_offset_diff); self.curr = Some(next.ch); @@ -346,8 +355,8 @@ impl<'a> StringReader<'a> { pub fn nextch(&self) -> Option<char> { let offset = self.byte_offset(self.pos).to_usize(); - if offset < self.filemap.src.len() { - Some(self.filemap.src.char_at(offset)) + if offset < self.source_text.len() { + Some(self.source_text.char_at(offset)) } else { None } @@ -359,7 +368,7 @@ impl<'a> StringReader<'a> { pub fn nextnextch(&self) -> Option<char> { let offset = self.byte_offset(self.pos).to_usize(); - let s = &*self.filemap.src; + let s = &self.source_text[..]; if offset >= s.len() { return None } let str::CharRange { next, .. } = s.char_range_at(offset); if next < s.len() { diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 4d099529cb4..66589d5e3d1 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -751,6 +751,7 @@ pub fn integer_lit(s: &str, suffix: Option<&str>, sd: &SpanHandler, sp: Span) -> #[cfg(test)] mod test { use super::*; + use std::rc::Rc; use serialize::json; use codemap::{Span, BytePos, Pos, Spanned, NO_EXPANSION}; use owned_slice::OwnedSlice; @@ -855,117 +856,50 @@ mod test { } #[test] - fn string_to_tts_1 () { + fn string_to_tts_1() { let tts = string_to_tts("fn a (b : i32) { b; }".to_string()); - assert_eq!(json::encode(&tts).unwrap(), - "[\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"Ident\",\ - \"fields\":[\ - \"fn\",\ - \"Plain\"\ - ]\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"Ident\",\ - \"fields\":[\ - \"a\",\ - \"Plain\"\ - ]\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtDelimited\",\ - \"fields\":[\ - null,\ - {\ - \"delim\":\"Paren\",\ - \"open_span\":null,\ - \"tts\":[\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"Ident\",\ - \"fields\":[\ - \"b\",\ - \"Plain\"\ - ]\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"Colon\"\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"Ident\",\ - \"fields\":[\ - \"i32\",\ - \"Plain\"\ - ]\ - }\ - ]\ - }\ - ],\ - \"close_span\":null\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtDelimited\",\ - \"fields\":[\ - null,\ - {\ - \"delim\":\"Brace\",\ - \"open_span\":null,\ - \"tts\":[\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"Ident\",\ - \"fields\":[\ - \"b\",\ - \"Plain\"\ - ]\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"Semi\"\ - ]\ - }\ - ],\ - \"close_span\":null\ - }\ - ]\ - }\ -]" - ); + + let expected = vec![ + ast::TtToken(sp(0, 2), + token::Ident(str_to_ident("fn"), + token::IdentStyle::Plain)), + ast::TtToken(sp(3, 4), + token::Ident(str_to_ident("a"), + token::IdentStyle::Plain)), + ast::TtDelimited( + sp(5, 14), + Rc::new(ast::Delimited { + delim: token::DelimToken::Paren, + open_span: sp(5, 6), + tts: vec![ + ast::TtToken(sp(6, 7), + token::Ident(str_to_ident("b"), + token::IdentStyle::Plain)), + ast::TtToken(sp(8, 9), + token::Colon), + ast::TtToken(sp(10, 13), + token::Ident(str_to_ident("i32"), + token::IdentStyle::Plain)), + ], + close_span: sp(13, 14), + })), + ast::TtDelimited( + sp(15, 21), + Rc::new(ast::Delimited { + delim: token::DelimToken::Brace, + open_span: sp(15, 16), + tts: vec![ + ast::TtToken(sp(17, 18), + token::Ident(str_to_ident("b"), + token::IdentStyle::Plain)), + ast::TtToken(sp(18, 19), + token::Semi) + ], + close_span: sp(20, 21), + })) + ]; + + assert_eq!(tts, expected); } #[test] fn ret_expr() { |