Commenting, test cases, cleanup

7 files changed, 118 insertions, 28 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index b109333c3b5..b26cab7694c 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -779,10 +779,19 @@ pub enum expr_ {
 #[auto_decode]
 #[doc="For macro invocations; parsing is delegated to the macro"]
 pub enum token_tree {
+    // a single token
     tt_tok(span, ::parse::token::Token),
+    // a delimited sequence (the delimiters appear as the first
+    // and last elements of the vector)
     tt_delim(~[token_tree]),
-    // These only make sense for right-hand-sides of MBE macros
+    // These only make sense for right-hand-sides of MBE macros:
+
+    // a kleene-style repetition sequence with a span, a tt_forest,
+    // an optional separator (?), and a boolean where true indicates
+    // zero or more (*), and false indicates one or more (+).
     tt_seq(span, ~[token_tree], Option<::parse::token::Token>, bool),
+
+    // a syntactic variable that will be filled in by macro expansion.
     tt_nonterminal(span, ident)
 }
 
diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 8c3db21f4ea..a85a7990ace 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -44,8 +44,8 @@ pub struct SyntaxExpanderTT {
     span: Option<span>
 }
 
-pub type SyntaxExpanderTTFun = fn@(ext_ctxt, span, ~[ast::token_tree])
-                                -> MacResult;
+pub type SyntaxExpanderTTFun
+    = fn@(ext_ctxt, span, ~[ast::token_tree]) -> MacResult;
 
 pub struct SyntaxExpanderTTItem {
     expander: SyntaxExpanderTTItemFun,
@@ -59,7 +59,7 @@ pub enum MacResult {
     MRExpr(@ast::expr),
     MRItem(@ast::item),
     MRAny(fn@()-> @ast::expr, fn@()-> Option<@ast::item>, fn@()->@ast::stmt),
-    MRDef(MacroDef)
+    MRDef(MacroDef),
 }
 
 pub enum SyntaxExtension {
@@ -78,9 +78,11 @@ pub enum SyntaxExtension {
 // A temporary hard-coded map of methods for expanding syntax extension
 // AST nodes into full ASTs
 pub fn syntax_expander_table() -> HashMap<~str, SyntaxExtension> {
+    // utility function to simplify creating NormalTT syntax extensions
     fn builtin_normal_tt(f: SyntaxExpanderTTFun) -> SyntaxExtension {
         NormalTT(SyntaxExpanderTT{expander: f, span: None})
     }
+    // utility function to simplify creating ItemTT syntax extensions
     fn builtin_item_tt(f: SyntaxExpanderTTItemFun) -> SyntaxExtension {
         ItemTT(SyntaxExpanderTTItem{expander: f, span: None})
     }
@@ -112,8 +114,8 @@ pub fn syntax_expander_table() -> HashMap<~str, SyntaxExtension> {
                                 ext::deriving::expand_deriving_iter_bytes));
 
     // Quasi-quoting expanders
-    syntax_expanders.insert(
-        ~"quote_tokens", builtin_normal_tt(ext::quote::expand_quote_tokens));
+    syntax_expanders.insert(~"quote_tokens",
+                       builtin_normal_tt(ext::quote::expand_quote_tokens));
     syntax_expanders.insert(~"quote_expr",
                             builtin_normal_tt(ext::quote::expand_quote_expr));
     syntax_expanders.insert(~"quote_ty",
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index aa9036d295e..75120e9c0bc 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -121,15 +121,15 @@ pure fn lookup_cur_matched_by_matched(r: @mut TtReader,
     vec::foldl(start, r.repeat_idx, red)
 }
 
-fn lookup_cur_matched(r: @mut TtReader, name: ident) -> @named_match {
+fn lookup_cur_matched(r: &TtReader, name: ident) -> @named_match {
     lookup_cur_matched_by_matched(r, r.interpolations.get(&name))
 }
 enum lis {
     lis_unconstrained, lis_constraint(uint, ident), lis_contradiction(~str)
 }
 
-fn lockstep_iter_size(t: token_tree, r: @mut TtReader) -> lis {
-    fn lis_merge(lhs: lis, rhs: lis, r: @mut TtReader) -> lis {
+fn lockstep_iter_size(t: token_tree, r: &TtReader) -> lis {
+    fn lis_merge(lhs: lis, rhs: lis, r: &TtReader) -> lis {
         match lhs {
           lis_unconstrained => rhs,
           lis_contradiction(_) => lhs,
diff --git a/src/libsyntax/parse/common.rs b/src/libsyntax/parse/common.rs
index e0d53fadfa0..7e74163b6bf 100644
--- a/src/libsyntax/parse/common.rs
+++ b/src/libsyntax/parse/common.rs
@@ -20,6 +20,8 @@ use core::option::{None, Option, Some};
 use core::option;
 use std::oldmap::HashMap;
 
+// seq_sep : a sequence separator (token)
+// and whether a trailing separator is allowed.
 pub type seq_sep = {
     sep: Option<token::Token>,
     trailing_sep_allowed: bool
@@ -51,6 +53,8 @@ pub impl Parser {
                    + token_to_str(self.reader, self.token) + ~"`");
     }
 
+    // expect and consume the token t. Signal an error if
+    // the next token is not t.
     fn expect(t: token::Token) {
         if self.token == t {
             self.bump();
@@ -88,6 +92,8 @@ pub impl Parser {
         return self.parse_ident();
     }
 
+    // consume token 'tok' if it exists. Returns true if the given
+    // token was present, false otherwise.
     fn eat(tok: token::Token) -> bool {
         return if self.token == tok { self.bump(); true } else { false };
     }
@@ -185,6 +191,8 @@ pub impl Parser {
         }
     }
 
+    // expect and consume a GT. if a >> is seen, replace it
+    // with a single > and continue.
     fn expect_gt() {
         if self.token == token::GT {
             self.bump();
@@ -202,16 +210,19 @@ pub impl Parser {
         }
     }
 
+    // parse a sequence bracketed by '<' and '>', stopping
+    // before the '>'.
     fn parse_seq_to_before_gt<T: Copy>(sep: Option<token::Token>,
                                        f: fn(Parser) -> T) -> ~[T] {
         let mut first = true;
         let mut v = ~[];
         while self.token != token::GT
+            // wait... isn't this going to eat a whole '>>' ?
             && self.token != token::BINOP(token::SHR) {
             match sep {
               Some(ref t) => {
                 if first { first = false; }
-                else { self.expect((*t)); }
+                else { self.expect(*t); }
               }
               _ => ()
             }
@@ -229,6 +240,7 @@ pub impl Parser {
         return v;
     }
 
+    // parse a sequence bracketed by '<' and '>'
     fn parse_seq_lt_gt<T: Copy>(sep: Option<token::Token>,
                                 f: fn(Parser) -> T) -> spanned<~[T]> {
         let lo = self.span.lo;
@@ -239,6 +251,9 @@ pub impl Parser {
         return spanned(lo, hi, result);
     }
 
+    // parse a sequence, including the closing delimiter. The function
+    // f must consume tokens until reaching the next separator or
+    // closing bracket.
     fn parse_seq_to_end<T: Copy>(ket: token::Token, sep: seq_sep,
                                  f: fn(Parser) -> T) -> ~[T] {
         let val = self.parse_seq_to_before_end(ket, sep, f);
@@ -246,7 +261,9 @@ pub impl Parser {
         return val;
     }
 
-
+    // parse a sequence, not including the closing delimiter. The function
+    // f must consume tokens until reaching the next separator or
+    // closing bracket.
     fn parse_seq_to_before_end<T: Copy>(ket: token::Token, sep: seq_sep,
                                         f: fn(Parser) -> T) -> ~[T] {
         let mut first: bool = true;
@@ -255,7 +272,7 @@ pub impl Parser {
             match sep.sep {
               Some(ref t) => {
                 if first { first = false; }
-                else { self.expect((*t)); }
+                else { self.expect(*t); }
               }
               _ => ()
             }
@@ -265,6 +282,9 @@ pub impl Parser {
         return v;
     }
 
+    // parse a sequence, including the closing delimiter. The function
+    // f must consume tokens until reaching the next separator or
+    // closing bracket.
     fn parse_unspanned_seq<T: Copy>(bra: token::Token,
                                     ket: token::Token,
                                     sep: seq_sep,
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index b8e671b3265..b863e2bd0e4 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -183,7 +183,6 @@ pub fn new_parser_from_file(sess: parse_sess,
           let srdr = lexer::new_string_reader(sess.span_diagnostic,
                                               filemap,
                                               sess.interner);
-
           Ok(Parser(sess, cfg, srdr as reader))
 
       }
@@ -222,3 +221,49 @@ pub fn new_parser_from_tts(sess: parse_sess, cfg: ast::crate_cfg,
     return Parser(sess, cfg, trdr as reader)
 }
 
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::serialize::Encodable;
+    use std;
+    use core::dvec;
+    use core::str;
+    use util::testing::*;
+
+    #[test] fn to_json_str (val: Encodable<std::json::Encoder>) -> ~str {
+        let bw = @io::BytesWriter {bytes: dvec::DVec(), pos: 0};
+        val.encode(~std::json::Encoder(bw as io::Writer));
+        str::from_bytes(bw.bytes.data)
+    }
+
+    #[test] fn alltts () {
+        let tts = parse_tts_from_source_str(
+            ~"bogofile",
+            @~"fn foo (x : int) { x; }",
+            ~[],
+            new_parse_sess(None));
+        check_equal(to_json_str(tts as Encodable::<std::json::Encoder>),
+                    //[["tt_tok",["IDENT","fn"]]]
+                    ~"abc"
+                   );
+        let ast1 = new_parser_from_tts(new_parse_sess(None),~[],tts)
+            .parse_item(~[]);
+        let ast2 = parse_item_from_source_str(
+            ~"bogofile",
+            @~"fn foo (x : int) { x; }",
+            ~[],~[],
+            new_parse_sess(None));
+        check_equal(ast1,ast2);
+    }
+}
+
+//
+// Local Variables:
+// mode: rust
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// End:
+//
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 6382413b081..3c1f306ff07 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -182,7 +182,8 @@ pure fn maybe_append(+lhs: ~[attribute], rhs: Option<~[attribute]>)
 
 /* ident is handled by common.rs */
 
-pub fn Parser(sess: parse_sess,
+pub fn Parser(sess: parse_sess
+              ,
               cfg: ast::crate_cfg,
               +rdr: reader) -> Parser {
 
@@ -1238,6 +1239,8 @@ pub impl Parser {
         return e;
     }
 
+    // parse an optional separator followed by a kleene-style
+    // repetition token (+ or *).
     fn parse_sep_and_zerok() -> (Option<token::Token>, bool) {
         if self.token == token::BINOP(token::STAR)
             || self.token == token::BINOP(token::PLUS) {
@@ -1258,20 +1261,18 @@ pub impl Parser {
         }
     }
 
+    // parse a single token tree from the input.
     fn parse_token_tree() -> token_tree {
         maybe_whole!(deref self, nt_tt);
 
-        fn parse_tt_tok(p: Parser, delim_ok: bool) -> token_tree {
+        fn parse_non_delim_tt_tok(p: Parser) -> token_tree {
             maybe_whole!(deref p, nt_tt);
             match p.token {
               token::RPAREN | token::RBRACE | token::RBRACKET
-              if !delim_ok => {
+              => {
                 p.fatal(~"incorrect close delimiter: `"
                            + token_to_str(p.reader, p.token) + ~"`");
               }
-              token::EOF => {
-                p.fatal(~"file ended in the middle of a macro invocation");
-              }
               /* we ought to allow different depths of unquotation */
               token::DOLLAR if p.quote_depth > 0u => {
                 p.bump();
@@ -1282,32 +1283,43 @@ pub impl Parser {
                                           seq_sep_none(),
                                           |p| p.parse_token_tree());
                     let (s, z) = p.parse_sep_and_zerok();
-                    return tt_seq(mk_sp(sp.lo ,p.span.hi), seq.node, s, z);
+                    tt_seq(mk_sp(sp.lo ,p.span.hi), seq.node, s, z)
                 } else {
-                    return tt_nonterminal(sp, p.parse_ident());
+                    tt_nonterminal(sp, p.parse_ident())
                 }
               }
-              _ => { /* ok */ }
+              _ => {
+                  parse_any_tt_tok(p)
+              }
             }
+        }
+
+        // turn the next token into a tt_tok:
+        fn parse_any_tt_tok(p: Parser) -> token_tree{
             let res = tt_tok(p.span, p.token);
             p.bump();
-            return res;
+            res
         }
 
-        return match self.token {
+        match self.token {
+          token::EOF => {
+                self.fatal(~"file ended in the middle of a macro invocation");
+          }
           token::LPAREN | token::LBRACE | token::LBRACKET => {
               // tjc: ??????
             let ket = token::flip_delimiter(copy self.token);
             tt_delim(vec::append(
-                ~[parse_tt_tok(self, true)],
+                // the open delimiter:
+                ~[parse_any_tt_tok(self)],
                 vec::append(
                     self.parse_seq_to_before_end(
                         ket, seq_sep_none(),
                         |p| p.parse_token_tree()),
-                    ~[parse_tt_tok(self, true)])))
+                    // the close delimiter:
+                    ~[parse_any_tt_tok(self)])))
           }
-          _ => parse_tt_tok(self, false)
-        };
+          _ => parse_non_delim_tt_tok(self)
+        }
     }
 
     fn parse_all_token_trees() -> ~[token_tree] {
@@ -3999,6 +4011,7 @@ pub impl Parser {
     }
 }
 
+
 //
 // Local Variables:
 // mode: rust
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index b8d756d893a..1f8b04630e2 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -86,6 +86,7 @@ pub enum Token {
     LIT_STR(ast::ident),
 
     /* Name components */
+    // an identifier contains an "is_mod_name" boolean.
     IDENT(ast::ident, bool),
     UNDERSCORE,
     LIFETIME(ast::ident),