Add Span and separate open/close delims to TTDelim

This came up when working [on the gl-rs generator extension](https://github.com/bjz/gl-rs/blob/990383de801bd2e233159d5be07c9b5622827620/src/gl_generator/lib.rs#L135-L146).

The new definition of  `TTDelim` adds an associated `Span` that covers the whole token tree and enforces the invariant that a delimited sequence of token trees must have an opening and closing delimiter.

A `get_span` method has also been added to `TokenTree` type to make it easier to implement better error messages for syntax extensions.

4 files changed, 50 insertions, 49 deletions

diff --git a/src/libsyntax/ext/log_syntax.rs b/src/libsyntax/ext/log_syntax.rs
index 8df5746e412..30301e3b8cc 100644
--- a/src/libsyntax/ext/log_syntax.rs
+++ b/src/libsyntax/ext/log_syntax.rs
@@ -13,16 +13,14 @@ use codemap;
 use ext::base;
 use print;
 
-use std::rc::Rc;
-
 pub fn expand_syntax_ext<'cx>(cx: &'cx mut base::ExtCtxt,
                               sp: codemap::Span,
-                              tt: &[ast::TokenTree])
+                              tts: &[ast::TokenTree])
                               -> Box<base::MacResult+'cx> {
 
     cx.print_backtrace();
-    println!("{}", print::pprust::tt_to_string(&ast::TTDelim(
-                Rc::new(tt.iter().map(|x| (*x).clone()).collect()))));
+
+    println!("{}", print::pprust::tts_to_string(tts));
 
     // any so that `log_syntax` can be invoked as an expression and item.
     base::DummyResult::any(sp)
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 84775c12d64..783c08a4443 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -637,7 +637,7 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
 }
 
 
-fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
+fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
     match *tt {
         ast::TTTok(sp, ref tok) => {
             let e_sp = cx.expr_ident(sp, id_ext("_sp"));
@@ -650,13 +650,16 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
                                     id_ext("push"),
                                     vec!(e_tok));
             vec!(cx.stmt_expr(e_push))
-        }
-
-        ast::TTDelim(ref tts) => mk_tts(cx, sp, tts.as_slice()),
+        },
+        ast::TTDelim(sp, ref open, ref tts, ref close) => {
+            let mut stmts = vec![];
+            stmts.extend(mk_tt(cx, sp, &open.to_tt()).into_iter());
+            stmts.extend(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter()));
+            stmts.extend(mk_tt(cx, sp, &close.to_tt()).into_iter());
+            stmts
+        },
         ast::TTSeq(..) => fail!("TTSeq in quote!"),
-
         ast::TTNonterminal(sp, ident) => {
-
             // tt.extend($ident.to_tokens(ext_cx).into_iter())
 
             let e_to_toks =
@@ -674,7 +677,7 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
                                     vec!(e_to_toks));
 
             vec!(cx.stmt_expr(e_push))
-        }
+        },
     }
 }
 
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 91db3a9d8df..fbfe10d004e 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -147,13 +147,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                           rhses: &[Rc<NamedMatch>])
                           -> Box<MacResult+'cx> {
     if cx.trace_macros() {
-        println!("{}! {} {} {}",
+        println!("{}! {{ {} }}",
                  token::get_ident(name),
-                 "{",
-                 print::pprust::tt_to_string(&TTDelim(Rc::new(arg.iter()
-                                                              .map(|x| (*x).clone())
-                                                              .collect()))),
-                 "}");
+                 print::pprust::tts_to_string(arg));
     }
 
     // Which arm's failure should we report? (the one furthest along)
@@ -175,15 +171,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                     // okay, what's your transcriber?
                     MatchedNonterminal(NtTT(ref tt)) => {
                         match **tt {
-                            // cut off delimiters; don't parse 'em
-                            TTDelim(ref tts) => {
-                                (*tts).slice(1u,(*tts).len()-1u)
-                                      .iter()
-                                      .map(|x| (*x).clone())
-                                      .collect()
-                            }
-                            _ => cx.span_fatal(
-                                sp, "macro rhs must be delimited")
+                            // ignore delimiters
+                            TTDelim(_, _, ref tts, _) => (**tts).clone(),
+                            _ => cx.span_fatal(sp, "macro rhs must be delimited"),
                         }
                     },
                     _ => cx.span_bug(sp, "bad thing in rhs")
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 35ec37d842a..472b24be81b 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -18,6 +18,7 @@ use parse::token;
 use parse::lexer::TokenAndSpan;
 
 use std::rc::Rc;
+use std::ops::Add;
 use std::collections::HashMap;
 
 ///an unzipping of `TokenTree`s
@@ -104,37 +105,41 @@ enum LockstepIterSize {
     LisContradiction(String),
 }
 
-fn lis_merge(lhs: LockstepIterSize, rhs: LockstepIterSize) -> LockstepIterSize {
-    match lhs {
-        LisUnconstrained => rhs.clone(),
-        LisContradiction(_) => lhs.clone(),
-        LisConstraint(l_len, l_id) => match rhs {
-            LisUnconstrained => lhs.clone(),
-            LisContradiction(_) => rhs.clone(),
-            LisConstraint(r_len, _) if l_len == r_len => lhs.clone(),
-            LisConstraint(r_len, r_id) => {
-                let l_n = token::get_ident(l_id);
-                let r_n = token::get_ident(r_id);
-                LisContradiction(format!("inconsistent lockstep iteration: \
-                                          '{}' has {} items, but '{}' has {}",
-                                          l_n, l_len, r_n, r_len).to_string())
-            }
+impl Add<LockstepIterSize, LockstepIterSize> for LockstepIterSize {
+    fn add(&self, other: &LockstepIterSize) -> LockstepIterSize {
+        match *self {
+            LisUnconstrained => other.clone(),
+            LisContradiction(_) => self.clone(),
+            LisConstraint(l_len, l_id) => match *other {
+                LisUnconstrained => self.clone(),
+                LisContradiction(_) => other.clone(),
+                LisConstraint(r_len, _) if l_len == r_len => self.clone(),
+                LisConstraint(r_len, r_id) => {
+                    let l_n = token::get_ident(l_id);
+                    let r_n = token::get_ident(r_id);
+                    LisContradiction(format!("inconsistent lockstep iteration: \
+                                              '{}' has {} items, but '{}' has {}",
+                                              l_n, l_len, r_n, r_len).to_string())
+                }
+            },
         }
     }
 }
 
 fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
     match *t {
-        TTDelim(ref tts) | TTSeq(_, ref tts, _, _) => {
-            tts.iter().fold(LisUnconstrained, |lis, tt| {
-                lis_merge(lis, lockstep_iter_size(tt, r))
+        // The opening and closing delimiters are both tokens, so they are
+        // treated as `LisUnconstrained`.
+        TTDelim(_, _, ref tts, _) | TTSeq(_, ref tts, _, _) => {
+            tts.iter().fold(LisUnconstrained, |size, tt| {
+                size + lockstep_iter_size(tt, r)
             })
-        }
+        },
         TTTok(..) => LisUnconstrained,
         TTNonterminal(_, name) => match *lookup_cur_matched(r, name) {
             MatchedNonterminal(_) => LisUnconstrained,
             MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name)
-        }
+        },
     }
 }
 
@@ -197,9 +202,14 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
             (*frame.forest)[frame.idx].clone()
         };
         match t {
-            TTDelim(tts) => {
+            TTDelim(_, open, delimed_tts, close) => {
+                let mut tts = vec![];
+                tts.push(open.to_tt());
+                tts.extend(delimed_tts.iter().map(|x| (*x).clone()));
+                tts.push(close.to_tt());
+
                 r.stack.push(TtFrame {
-                    forest: tts,
+                    forest: Rc::new(tts),
                     idx: 0,
                     dotdotdoted: false,
                     sep: None