Encode codemap and span information in crate metadata.

This allows to create proper debuginfo line information for items inlined from other crates (e.g. instantiations of generics).
Only the codemap's 'metadata' is stored in a crate's metadata. That is, just filename, line-beginnings, etc. but not the actual source code itself. We are thus missing the opportunity of making Rust the first "open-source-only" programming language out there. Pity.

2 files changed, 65 insertions, 122 deletions

diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index bbe1ddfd4cf..8d3e93d35dd 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -76,6 +76,10 @@ pub struct StringReader<'a> {
     // are revised to go directly to token-trees.
     /// Is \x00<name>,<ctxt>\x00 is interpreted as encoded ast::Ident?
     read_embedded_ident: bool,
+
+    // cache a direct reference to the source text, so that we don't have to
+    // retrieve it via `self.filemap.src.as_ref().unwrap()` all the time.
+    source_text: Rc<String>
 }
 
 impl<'a> Reader for StringReader<'a> {
@@ -141,7 +145,14 @@ pub fn make_reader_with_embedded_idents<'b>(span_diagnostic: &'b SpanHandler,
 impl<'a> StringReader<'a> {
     /// For comments.rs, which hackily pokes into pos and curr
     pub fn new_raw<'b>(span_diagnostic: &'b SpanHandler,
-                   filemap: Rc<codemap::FileMap>) -> StringReader<'b> {
+                       filemap: Rc<codemap::FileMap>) -> StringReader<'b> {
+        if filemap.src.is_none() {
+            span_diagnostic.handler.bug(&format!("Cannot lex filemap without source: {}",
+                                                 filemap.name)[..]);
+        }
+
+        let source_text = (*filemap.src.as_ref().unwrap()).clone();
+
         let mut sr = StringReader {
             span_diagnostic: span_diagnostic,
             pos: filemap.start_pos,
@@ -153,6 +164,7 @@ impl<'a> StringReader<'a> {
             peek_tok: token::Eof,
             peek_span: codemap::DUMMY_SP,
             read_embedded_ident: false,
+            source_text: source_text
         };
         sr.bump();
         sr
@@ -213,7 +225,7 @@ impl<'a> StringReader<'a> {
         m.push_str(": ");
         let from = self.byte_offset(from_pos).to_usize();
         let to = self.byte_offset(to_pos).to_usize();
-        m.push_str(&self.filemap.src[from..to]);
+        m.push_str(&self.source_text[from..to]);
         self.fatal_span_(from_pos, to_pos, &m[..]);
     }
 
@@ -270,9 +282,8 @@ impl<'a> StringReader<'a> {
     fn with_str_from_to<T, F>(&self, start: BytePos, end: BytePos, f: F) -> T where
         F: FnOnce(&str) -> T,
     {
-        f(&self.filemap.src[
-                self.byte_offset(start).to_usize()..
-                self.byte_offset(end).to_usize()])
+        f(&self.source_text[self.byte_offset(start).to_usize()..
+                            self.byte_offset(end).to_usize()])
     }
 
     /// Converts CRLF to LF in the given string, raising an error on bare CR.
@@ -321,12 +332,10 @@ impl<'a> StringReader<'a> {
     pub fn bump(&mut self) {
         self.last_pos = self.pos;
         let current_byte_offset = self.byte_offset(self.pos).to_usize();
-        if current_byte_offset < self.filemap.src.len() {
+        if current_byte_offset < self.source_text.len() {
             assert!(self.curr.is_some());
             let last_char = self.curr.unwrap();
-            let next = self.filemap
-                          .src
-                          .char_range_at(current_byte_offset);
+            let next = self.source_text.char_range_at(current_byte_offset);
             let byte_offset_diff = next.next - current_byte_offset;
             self.pos = self.pos + Pos::from_usize(byte_offset_diff);
             self.curr = Some(next.ch);
@@ -346,8 +355,8 @@ impl<'a> StringReader<'a> {
 
     pub fn nextch(&self) -> Option<char> {
         let offset = self.byte_offset(self.pos).to_usize();
-        if offset < self.filemap.src.len() {
-            Some(self.filemap.src.char_at(offset))
+        if offset < self.source_text.len() {
+            Some(self.source_text.char_at(offset))
         } else {
             None
         }
@@ -359,7 +368,7 @@ impl<'a> StringReader<'a> {
 
     pub fn nextnextch(&self) -> Option<char> {
         let offset = self.byte_offset(self.pos).to_usize();
-        let s = &*self.filemap.src;
+        let s = &self.source_text[..];
         if offset >= s.len() { return None }
         let str::CharRange { next, .. } = s.char_range_at(offset);
         if next < s.len() {
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 4d099529cb4..66589d5e3d1 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -751,6 +751,7 @@ pub fn integer_lit(s: &str, suffix: Option<&str>, sd: &SpanHandler, sp: Span) ->
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::rc::Rc;
     use serialize::json;
     use codemap::{Span, BytePos, Pos, Spanned, NO_EXPANSION};
     use owned_slice::OwnedSlice;
@@ -855,117 +856,50 @@ mod test {
     }
 
     #[test]
-    fn string_to_tts_1 () {
+    fn string_to_tts_1() {
         let tts = string_to_tts("fn a (b : i32) { b; }".to_string());
-        assert_eq!(json::encode(&tts).unwrap(),
-        "[\
-    {\
-        \"variant\":\"TtToken\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"variant\":\"Ident\",\
-                \"fields\":[\
-                    \"fn\",\
-                    \"Plain\"\
-                ]\
-            }\
-        ]\
-    },\
-    {\
-        \"variant\":\"TtToken\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"variant\":\"Ident\",\
-                \"fields\":[\
-                    \"a\",\
-                    \"Plain\"\
-                ]\
-            }\
-        ]\
-    },\
-    {\
-        \"variant\":\"TtDelimited\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"delim\":\"Paren\",\
-                \"open_span\":null,\
-                \"tts\":[\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            {\
-                                \"variant\":\"Ident\",\
-                                \"fields\":[\
-                                    \"b\",\
-                                    \"Plain\"\
-                                ]\
-                            }\
-                        ]\
-                    },\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            \"Colon\"\
-                        ]\
-                    },\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            {\
-                                \"variant\":\"Ident\",\
-                                \"fields\":[\
-                                    \"i32\",\
-                                    \"Plain\"\
-                                ]\
-                            }\
-                        ]\
-                    }\
-                ],\
-                \"close_span\":null\
-            }\
-        ]\
-    },\
-    {\
-        \"variant\":\"TtDelimited\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"delim\":\"Brace\",\
-                \"open_span\":null,\
-                \"tts\":[\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            {\
-                                \"variant\":\"Ident\",\
-                                \"fields\":[\
-                                    \"b\",\
-                                    \"Plain\"\
-                                ]\
-                            }\
-                        ]\
-                    },\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            \"Semi\"\
-                        ]\
-                    }\
-                ],\
-                \"close_span\":null\
-            }\
-        ]\
-    }\
-]"
-        );
+
+        let expected = vec![
+            ast::TtToken(sp(0, 2),
+                         token::Ident(str_to_ident("fn"),
+                         token::IdentStyle::Plain)),
+            ast::TtToken(sp(3, 4),
+                         token::Ident(str_to_ident("a"),
+                         token::IdentStyle::Plain)),
+            ast::TtDelimited(
+                sp(5, 14),
+                Rc::new(ast::Delimited {
+                    delim: token::DelimToken::Paren,
+                    open_span: sp(5, 6),
+                    tts: vec![
+                        ast::TtToken(sp(6, 7),
+                                     token::Ident(str_to_ident("b"),
+                                     token::IdentStyle::Plain)),
+                        ast::TtToken(sp(8, 9),
+                                     token::Colon),
+                        ast::TtToken(sp(10, 13),
+                                     token::Ident(str_to_ident("i32"),
+                                     token::IdentStyle::Plain)),
+                    ],
+                    close_span: sp(13, 14),
+                })),
+            ast::TtDelimited(
+                sp(15, 21),
+                Rc::new(ast::Delimited {
+                    delim: token::DelimToken::Brace,
+                    open_span: sp(15, 16),
+                    tts: vec![
+                        ast::TtToken(sp(17, 18),
+                                     token::Ident(str_to_ident("b"),
+                                     token::IdentStyle::Plain)),
+                        ast::TtToken(sp(18, 19),
+                                     token::Semi)
+                    ],
+                    close_span: sp(20, 21),
+                }))
+        ];
+
+        assert_eq!(tts, expected);
     }
 
     #[test] fn ret_expr() {