Allow defining token tree macros. They should work now!

3 files changed, 124 insertions, 32 deletions

diff --git a/src/libsyntax/ext/tt/earley_parser.rs b/src/libsyntax/ext/tt/earley_parser.rs
index f890661e239..d26f5dc0e3c 100644
--- a/src/libsyntax/ext/tt/earley_parser.rs
+++ b/src/libsyntax/ext/tt/earley_parser.rs
@@ -72,11 +72,10 @@ enum arb_depth { leaf(whole_nt), seq(~[@arb_depth], codemap::span) }
 
 type earley_item = matcher_pos;
 
-
-fn nameize(&&p_s: parse_sess, ms: ~[matcher], &&res: ~[@arb_depth])
+fn nameize(p_s: parse_sess, ms: ~[matcher], res: ~[@arb_depth])
     -> hashmap<ident,@arb_depth> {
-    fn n_rec(&&p_s: parse_sess, &&m: matcher, &&res: ~[@arb_depth],
-             &&ret_val: hashmap<ident, @arb_depth>) {
+    fn n_rec(p_s: parse_sess, m: matcher, res: ~[@arb_depth],
+             ret_val: hashmap<ident, @arb_depth>) {
         alt m {
           {node: mtc_tok(_), span: _} { }
           {node: mtc_rep(more_ms, _, _), span: _} {
@@ -142,8 +141,11 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
                         // doing a lot of array work that will get thrown away
                         // most of the time.
                         for ei.matches.eachi() |idx, elt| {
+                            let sub = elt.get();
+                            // Some subtrees don't contain the name at all
+                            if sub.len() == 0u { cont; }
                             new_pos.matches[idx]
-                                .push(@seq(elt.get(), mk_sp(ei.sp_lo,sp.hi)));
+                                .push(@seq(sub, mk_sp(ei.sp_lo,sp.hi)));
                         }
 
                         new_pos.idx += 1u;
@@ -221,8 +223,8 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
                      built-in NTs %s or %u other options.",
                     nts, next_eis.len()]);
             } else if (bb_eis.len() == 0u && next_eis.len() == 0u) {
-                failure(sp, "No rules expected the token "
-                        + to_str(*rdr.interner(), tok));
+                ret failure(sp, "No rules expected the token "
+                            + to_str(*rdr.interner(), tok));
             } else if (next_eis.len() > 0u) {
                 /* Now process the next token */
                 while(next_eis.len() > 0u) {
@@ -246,6 +248,9 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
                 /* this would fail if zero-length tokens existed */
                 while rdr.peek().sp.lo < rust_parser.span.lo {
                     rdr.next_token();
+                } /* except for EOF... */
+                while rust_parser.token == EOF && rdr.peek().tok != EOF {
+                    rdr.next_token();
                 }
             }
         }
@@ -273,7 +278,7 @@ fn parse_nt(p: parser, name: str) -> whole_nt {
       } }
       "path" { token::w_path(p.parse_path_with_tps(false)) }
       "tt" {
-        p.quote_depth += 1u;
+        p.quote_depth += 1u; //but in theory, non-quoted tts might be useful
         let res = token::w_tt(@p.parse_token_tree());
         p.quote_depth -= 1u;
         res
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
new file mode 100644
index 00000000000..822e8e0d697
--- /dev/null
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -0,0 +1,83 @@
+import base::{ext_ctxt, mac_result, mr_expr, mr_def, expr_tt};
+import codemap::span;
+import ast::{ident, matcher_, matcher, mtc_tok, mtc_bb, mtc_rep, tt_delim};
+import parse::lexer::{new_tt_reader, tt_reader_as_reader, reader};
+import parse::token::{FAT_ARROW, SEMI, LBRACE, RBRACE, w_mtcs, w_tt};
+import parse::parser::{parser, SOURCE_FILE};
+import earley_parser::{parse, success, failure, arb_depth, seq, leaf};
+import std::map::hashmap;
+
+
+
+fn add_new_extension(cx: ext_ctxt, sp: span, name: ident,
+                     arg: ~[ast::token_tree]) -> base::mac_result {
+    // these spans won't matter, anyways
+    fn ms(m: matcher_) -> matcher {
+        {node: m, span: {lo: 0u, hi: 0u, expn_info: none}}
+    }
+
+    let argument_gram = ~[
+        ms(mtc_rep(~[
+            ms(mtc_bb(@"lhs",@"mtcs", 0u)),
+            ms(mtc_tok(FAT_ARROW)),
+            ms(mtc_bb(@"rhs",@"tt", 1u)),
+        ], some(SEMI), false))];
+
+    let arg_reader = new_tt_reader(cx.parse_sess().span_diagnostic,
+                                   cx.parse_sess().interner, none, arg);
+    let arguments = alt parse(cx.parse_sess(), cx.cfg(),
+                              arg_reader as reader, argument_gram) {
+      success(m) { m }
+      failure(sp, msg) { cx.span_fatal(sp, msg); }
+    };
+
+    let lhses = alt arguments.get(@"lhs") {
+      @seq(s, sp) { s }
+      _ { cx.span_bug(sp, "wrong-structured lhs") }
+    };
+    let rhses = alt arguments.get(@"rhs") {
+      @seq(s, sp) { s }
+      _ { cx.span_bug(sp, "wrong-structured rhs") }
+    };
+
+    fn generic_extension(cx: ext_ctxt, sp: span, arg: ~[ast::token_tree],
+                         lhses: ~[@arb_depth], rhses: ~[@arb_depth])
+    -> mac_result {
+        let mut best_fail_spot = {lo: 0u, hi: 0u, expn_info: none};
+        let mut best_fail_msg = "internal error: ran no matchers";
+
+        let s_d = cx.parse_sess().span_diagnostic;
+        let itr = cx.parse_sess().interner;
+
+        for lhses.eachi() |i, lhs| {
+            alt lhs {
+              @leaf(w_mtcs(mtcs)) {
+                let arg_rdr = new_tt_reader(s_d, itr, none, arg) as reader;
+                alt parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtcs) {
+                  success(m) {
+                    let rhs = alt rhses[i] {
+                      @leaf(w_tt(@tt)) { tt }
+                      _ { cx.span_bug(sp, "bad thing in rhs") }
+                    };
+                    let trncbr = new_tt_reader(s_d, itr, some(m), ~[rhs]);
+                    let p = parser(cx.parse_sess(), cx.cfg(),
+                                   trncbr as reader, SOURCE_FILE);
+                    ret mr_expr(p.parse_expr());
+                  }
+                  failure(sp, msg) {
+                    if sp.lo >= best_fail_spot.lo {
+                        best_fail_spot = sp; best_fail_msg = msg;
+                    }
+                  }
+                }
+              }
+              _ { cx.bug("non-matcher found in parsed lhses"); }
+            }
+        }
+        cx.span_fatal(best_fail_spot, best_fail_msg);
+    }
+
+    let exp = |cx, sp, arg| generic_extension(cx, sp, arg, lhses, rhses);
+
+    ret mr_def({ident: name, ext: expr_tt({expander: exp, span: some(sp)})});
+}
\ No newline at end of file
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index e84bc5c5421..8924c5820a9 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -23,7 +23,7 @@ type tt_frame = @{
 };
 
 type tt_reader = @{
-    span_diagnostic: span_handler,
+    sp_diag: span_handler,
     interner: @interner<@str>,
     mut cur: tt_frame,
     /* for MBE-style macro transcription */
@@ -36,13 +36,13 @@ type tt_reader = @{
 };
 
 /** This can do Macro-By-Example transcription. On the other hand, if
- *  `doc` contains no `tt_dotdotdot`s and `tt_interpolate`s, `interp` can (and
+ *  `src` contains no `tt_dotdotdot`s and `tt_interpolate`s, `interp` can (and
  *  should) be none. */
-fn new_tt_reader(span_diagnostic: span_handler, itr: @interner<@str>,
+fn new_tt_reader(sp_diag: span_handler, itr: @interner<@str>,
                  interp: option<std::map::hashmap<ident,@arb_depth>>,
                  src: ~[ast::token_tree])
     -> tt_reader {
-    let r = @{span_diagnostic: span_diagnostic, interner: itr,
+    let r = @{sp_diag: sp_diag, interner: itr,
               mut cur: @{readme: src, mut idx: 0u, dotdotdoted: false,
                          sep: none, up: tt_frame_up(option::none)},
               interpolations: alt interp { /* just a convienience */
@@ -70,7 +70,7 @@ pure fn dup_tt_frame(&&f: tt_frame) -> tt_frame {
 }
 
 pure fn dup_tt_reader(&&r: tt_reader) -> tt_reader {
-    @{span_diagnostic: r.span_diagnostic, interner: r.interner,
+    @{sp_diag: r.sp_diag, interner: r.interner,
       mut cur: dup_tt_frame(r.cur),
       interpolations: r.interpolations,
       mut repeat_idx: copy r.repeat_idx, mut repeat_len: copy r.repeat_len,
@@ -132,28 +132,27 @@ fn lockstep_iter_size(&&t: token_tree, &&r: tt_reader) -> lis {
 
 fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
     let ret_val = { tok: r.cur_tok, sp: r.cur_span };
-    if r.cur.idx >= vec::len(r.cur.readme) {
+    while r.cur.idx >= vec::len(r.cur.readme) {
         /* done with this set; pop or repeat? */
         if ! r.cur.dotdotdoted
             || r.repeat_idx.last() == r.repeat_len.last() - 1 {
-            if r.cur.dotdotdoted {
-                vec::pop(r.repeat_idx); vec::pop(r.repeat_len);
-            }
+
             alt r.cur.up {
               tt_frame_up(none) {
                 r.cur_tok = EOF;
                 ret ret_val;
               }
               tt_frame_up(some(tt_f)) {
+                if r.cur.dotdotdoted {
+                    vec::pop(r.repeat_idx); vec::pop(r.repeat_len);
+                }
+
                 r.cur = tt_f;
-                /* the outermost `if` would need to be a `while` if we
-                didn't know that the last thing in a `tt_delim` is always
-                a `tt_flat`, and that a `tt_dotdotdot` is never empty */
                 r.cur.idx += 1u;
               }
             }
 
-        } else {
+        } else { /* repeat */
             r.cur.idx = 0u;
             r.repeat_idx[r.repeat_idx.len() - 1u] += 1u;
             alt r.cur.sep {
@@ -165,14 +164,13 @@ fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
             }
         }
     }
-    /* if `tt_delim`s could be 0-length, we'd need to be able to switch
-    between popping and pushing until we got to an actual `tt_flat` */
     loop { /* because it's easiest, this handles `tt_delim` not starting
     with a `tt_flat`, even though it won't happen */
         alt r.cur.readme[r.cur.idx] {
           tt_delim(tts) {
             r.cur = @{readme: tts, mut idx: 0u, dotdotdoted: false,
                       sep: none, up: tt_frame_up(option::some(r.cur)) };
+            // if this could be 0-length, we'd need to potentially recur here
           }
           tt_flat(sp, tok) {
             r.cur_span = sp; r.cur_tok = tok;
@@ -182,23 +180,29 @@ fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
           tt_dotdotdot(sp, tts, sep, zerok) {
             alt lockstep_iter_size(tt_dotdotdot(sp, tts, sep, zerok), r) {
               lis_unconstrained {
-                r.span_diagnostic.span_fatal(
-                    copy r.cur_span, /* blame macro writer */
+                r.sp_diag.span_fatal(
+                    sp, /* blame macro writer */
                     "attempted to repeat an expression containing no syntax \
                      variables matched as repeating at this depth");
               }
-              lis_contradiction(msg) { /* blame macro invoker */
-                r.span_diagnostic.span_fatal(sp, msg);
+              lis_contradiction(msg) { /* TODO blame macro invoker instead*/
+                r.sp_diag.span_fatal(sp, msg);
               }
               lis_constraint(len, _) {
-                if len == 0 && !zerok {
-                    r.span_diagnostic.span_fatal(sp, "this must repeat \
-                                                      at least once");
-                }
                 vec::push(r.repeat_len, len);
                 vec::push(r.repeat_idx, 0u);
                 r.cur = @{readme: tts, mut idx: 0u, dotdotdoted: true,
                           sep: sep, up: tt_frame_up(option::some(r.cur)) };
+
+                if len == 0 {
+                    if !zerok {
+                        r.sp_diag.span_fatal(sp, /* TODO blame invoker */
+                                             "this must repeat at least \
+                                              once");
+                    }
+                    /* we need to pop before we proceed, so recur */
+                    ret tt_next_token(r);
+                }
               }
             }
           }
@@ -219,7 +223,7 @@ fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
                 ret ret_val;
               }
               seq(*) {
-                r.span_diagnostic.span_fatal(
+                r.sp_diag.span_fatal(
                     copy r.cur_span, /* blame the macro writer */
                     #fmt["variable '%s' is still repeating at this depth",
                          *ident]);