Merge remote-tracking branch 'remotes/origin/incoming' into incoming

1 files changed, 49 insertions, 1 deletions

diff --git a/src/libsyntax/parse/lexer.rs b/src/libsyntax/parse/lexer.rs
index 54fdcc647ea..2a40b700c11 100644
--- a/src/libsyntax/parse/lexer.rs
+++ b/src/libsyntax/parse/lexer.rs
@@ -133,7 +133,7 @@ impl reader for StringReader {
     fn dup(@mut self) -> reader { dup_string_reader(self) as reader }
 }
 
-pub impl reader for TtReader {
+impl reader for TtReader {
     fn is_eof(@mut self) -> bool { self.cur_tok == token::EOF }
     fn next_token(@mut self) -> TokenAndSpan { tt_next_token(self) }
     fn fatal(@mut self, m: ~str) -> ! {
@@ -779,11 +779,13 @@ pub mod test {
     use diagnostic;
     use util::testing::{check_equal, check_equal_ptr};
 
+    // represents a testing reader (incl. both reader and interner)
     struct Env {
         interner: @token::ident_interner,
         string_reader: @mut StringReader
     }
 
+    // open a string reader for the given string
     fn setup(teststr: ~str) -> Env {
         let cm = CodeMap::new();
         let fm = cm.new_filemap(~"zebra.rs", @teststr);
@@ -818,6 +820,52 @@ pub mod test {
         check_equal (string_reader.last_pos,BytePos(29))
     }
 
+    // check that the given reader produces the desired stream
+    // of tokens (stop checking after exhausting the expected vec)
+    fn check_tokenization (env: Env, expected: ~[token::Token]) {
+        for expected.each |expected_tok| {
+            let TokenAndSpan {tok:actual_tok, sp: _} =
+                env.string_reader.next_token();
+            check_equal(&actual_tok,expected_tok);
+        }
+    }
+
+    // make the identifier by looking up the string in the interner
+    fn mk_ident (env: Env, id: ~str, is_mod_name: bool) -> token::Token {
+        token::IDENT (env.interner.intern(@id),is_mod_name)
+    }
+
+    #[test] fn doublecolonparsing () {
+        let env = setup (~"a b");
+        check_tokenization (env,
+                           ~[mk_ident (env,~"a",false),
+                             mk_ident (env,~"b",false)]);
+    }
+
+    #[test] fn dcparsing_2 () {
+        let env = setup (~"a::b");
+        check_tokenization (env,
+                           ~[mk_ident (env,~"a",true),
+                             token::MOD_SEP,
+                             mk_ident (env,~"b",false)]);
+    }
+
+    #[test] fn dcparsing_3 () {
+        let env = setup (~"a ::b");
+        check_tokenization (env,
+                           ~[mk_ident (env,~"a",false),
+                             token::MOD_SEP,
+                             mk_ident (env,~"b",false)]);
+    }
+
+    #[test] fn dcparsing_4 () {
+        let env = setup (~"a:: b");
+        check_tokenization (env,
+                           ~[mk_ident (env,~"a",true),
+                             token::MOD_SEP,
+                             mk_ident (env,~"b",false)]);
+    }
+
     #[test] fn character_a() {
         let env = setup(~"'a'");
         let TokenAndSpan {tok, sp: _} =