auto merge of #18368 : alexcrichton/rust/rollup, r=alexcrichton

9 files changed, 88 insertions, 78 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 5cc2fe03618..64c8068607a 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -684,8 +684,8 @@ pub fn get_single_str_from_tts(cx: &ExtCtxt,
         cx.span_err(sp, format!("{} takes 1 argument.", name).as_slice());
     } else {
         match tts[0] {
-            ast::TTTok(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())),
-            ast::TTTok(_, token::LIT_STR_RAW(ident, _)) => {
+            ast::TtToken(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())),
+            ast::TtToken(_, token::LIT_STR_RAW(ident, _)) => {
                 return Some(parse::raw_str_lit(ident.as_str()))
             }
             _ => {
diff --git a/src/libsyntax/ext/concat_idents.rs b/src/libsyntax/ext/concat_idents.rs
index 145412caa0b..e12f9ee133a 100644
--- a/src/libsyntax/ext/concat_idents.rs
+++ b/src/libsyntax/ext/concat_idents.rs
@@ -23,7 +23,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]
     for (i, e) in tts.iter().enumerate() {
         if i & 1 == 1 {
             match *e {
-                ast::TTTok(_, token::COMMA) => (),
+                ast::TtToken(_, token::COMMA) => (),
                 _ => {
                     cx.span_err(sp, "concat_idents! expecting comma.");
                     return DummyResult::expr(sp);
@@ -31,7 +31,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]
             }
         } else {
             match *e {
-                ast::TTTok(_, token::IDENT(ident,_)) => {
+                ast::TtToken(_, token::IDENT(ident,_)) => {
                     res_str.push_str(token::get_ident(ident).get())
                 }
                 _ => {
diff --git a/src/libsyntax/ext/deriving/generic/mod.rs b/src/libsyntax/ext/deriving/generic/mod.rs
index 533a28998bd..7c32b845508 100644
--- a/src/libsyntax/ext/deriving/generic/mod.rs
+++ b/src/libsyntax/ext/deriving/generic/mod.rs
@@ -1250,7 +1250,7 @@ impl<'a> TraitDef<'a> {
                 // id is guaranteed to be Some
                 codemap::Spanned {
                     span: pat.span,
-                    node: ast::FieldPat { ident: id.unwrap(), pat: pat, is_shorthand: true },
+                    node: ast::FieldPat { ident: id.unwrap(), pat: pat, is_shorthand: false },
                 }
             }).collect();
             cx.pat_struct(self.span, matching_path, field_pats)
diff --git a/src/libsyntax/ext/log_syntax.rs b/src/libsyntax/ext/log_syntax.rs
index 8df5746e412..30301e3b8cc 100644
--- a/src/libsyntax/ext/log_syntax.rs
+++ b/src/libsyntax/ext/log_syntax.rs
@@ -13,16 +13,14 @@ use codemap;
 use ext::base;
 use print;
 
-use std::rc::Rc;
-
 pub fn expand_syntax_ext<'cx>(cx: &'cx mut base::ExtCtxt,
                               sp: codemap::Span,
-                              tt: &[ast::TokenTree])
+                              tts: &[ast::TokenTree])
                               -> Box<base::MacResult+'cx> {
 
     cx.print_backtrace();
-    println!("{}", print::pprust::tt_to_string(&ast::TTDelim(
-                Rc::new(tt.iter().map(|x| (*x).clone()).collect()))));
+
+    println!("{}", print::pprust::tts_to_string(tts));
 
     // any so that `log_syntax` can be invoked as an expression and item.
     base::DummyResult::any(sp)
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 84775c12d64..6f1fd90adfa 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -23,7 +23,7 @@ use ptr::P;
 *
 * This is registered as a set of expression syntax extension called quote!
 * that lifts its argument token-tree to an AST representing the
-* construction of the same token tree, with ast::TTNonterminal nodes
+* construction of the same token tree, with ast::TtNonterminal nodes
 * interpreted as antiquotes (splices).
 *
 */
@@ -637,12 +637,12 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
 }
 
 
-fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
+fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
     match *tt {
-        ast::TTTok(sp, ref tok) => {
+        ast::TtToken(sp, ref tok) => {
             let e_sp = cx.expr_ident(sp, id_ext("_sp"));
             let e_tok = cx.expr_call(sp,
-                                     mk_ast_path(cx, sp, "TTTok"),
+                                     mk_ast_path(cx, sp, "TtToken"),
                                      vec!(e_sp, mk_token(cx, sp, tok)));
             let e_push =
                 cx.expr_method_call(sp,
@@ -650,13 +650,16 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
                                     id_ext("push"),
                                     vec!(e_tok));
             vec!(cx.stmt_expr(e_push))
-        }
-
-        ast::TTDelim(ref tts) => mk_tts(cx, sp, tts.as_slice()),
-        ast::TTSeq(..) => fail!("TTSeq in quote!"),
-
-        ast::TTNonterminal(sp, ident) => {
-
+        },
+        ast::TtDelimited(sp, ref delimed) => {
+            let (ref open, ref tts, ref close) = **delimed;
+            mk_tt(cx, sp, &open.to_tt()).into_iter()
+                .chain(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter()))
+                .chain(mk_tt(cx, sp, &close.to_tt()).into_iter())
+                .collect()
+        },
+        ast::TtSequence(..) => fail!("TtSequence in quote!"),
+        ast::TtNonterminal(sp, ident) => {
             // tt.extend($ident.to_tokens(ext_cx).into_iter())
 
             let e_to_toks =
@@ -674,7 +677,7 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
                                     vec!(e_to_toks));
 
             vec!(cx.stmt_expr(e_push))
-        }
+        },
     }
 }
 
@@ -690,7 +693,7 @@ fn mk_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
 fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
               -> (P<ast::Expr>, P<ast::Expr>) {
     // NB: It appears that the main parser loses its mind if we consider
-    // $foo as a TTNonterminal during the main parse, so we have to re-parse
+    // $foo as a TtNonterminal during the main parse, so we have to re-parse
     // under quote_depth > 0. This is silly and should go away; the _guess_ is
     // it has to do with transition away from supporting old-style macros, so
     // try removing it when enough of them are gone.
diff --git a/src/libsyntax/ext/trace_macros.rs b/src/libsyntax/ext/trace_macros.rs
index 1f50eb933bb..abf798ddacb 100644
--- a/src/libsyntax/ext/trace_macros.rs
+++ b/src/libsyntax/ext/trace_macros.rs
@@ -20,10 +20,10 @@ pub fn expand_trace_macros(cx: &mut ExtCtxt,
                            tt: &[ast::TokenTree])
                            -> Box<base::MacResult+'static> {
     match tt {
-        [ast::TTTok(_, ref tok)] if is_keyword(keywords::True, tok) => {
+        [ast::TtToken(_, ref tok)] if is_keyword(keywords::True, tok) => {
             cx.set_trace_macros(true);
         }
-        [ast::TTTok(_, ref tok)] if is_keyword(keywords::False, tok) => {
+        [ast::TtToken(_, ref tok)] if is_keyword(keywords::False, tok) => {
             cx.set_trace_macros(false);
         }
         _ => cx.span_err(sp, "trace_macros! accepts only `true` or `false`"),
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index f2081674fb7..cea8cab5265 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -323,9 +323,9 @@ pub fn parse(sess: &ParseSess,
             } else {
                 match ei.elts[idx].node.clone() {
                   /* need to descend into sequence */
-                  MatchSeq(ref matchers, ref sep, zero_ok,
+                  MatchSeq(ref matchers, ref sep, kleene_op,
                            match_idx_lo, match_idx_hi) => {
-                    if zero_ok {
+                    if kleene_op == ast::ZeroOrMore {
                         let mut new_ei = ei.clone();
                         new_ei.idx += 1u;
                         //we specifically matched zero repeats.
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 91db3a9d8df..3b51fb380b8 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TTDelim};
+use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TtDelimited};
 use ast;
 use codemap::{Span, Spanned, DUMMY_SP};
 use ext::base::{ExtCtxt, MacResult, MacroDef};
@@ -147,13 +147,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                           rhses: &[Rc<NamedMatch>])
                           -> Box<MacResult+'cx> {
     if cx.trace_macros() {
-        println!("{}! {} {} {}",
+        println!("{}! {{ {} }}",
                  token::get_ident(name),
-                 "{",
-                 print::pprust::tt_to_string(&TTDelim(Rc::new(arg.iter()
-                                                              .map(|x| (*x).clone())
-                                                              .collect()))),
-                 "}");
+                 print::pprust::tts_to_string(arg));
     }
 
     // Which arm's failure should we report? (the one furthest along)
@@ -175,15 +171,12 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                     // okay, what's your transcriber?
                     MatchedNonterminal(NtTT(ref tt)) => {
                         match **tt {
-                            // cut off delimiters; don't parse 'em
-                            TTDelim(ref tts) => {
-                                (*tts).slice(1u,(*tts).len()-1u)
-                                      .iter()
-                                      .map(|x| (*x).clone())
-                                      .collect()
-                            }
-                            _ => cx.span_fatal(
-                                sp, "macro rhs must be delimited")
+                            // ignore delimiters
+                            TtDelimited(_, ref delimed) => {
+                                let (_, ref tts, _) = **delimed;
+                                tts.clone()
+                            },
+                            _ => cx.span_fatal(sp, "macro rhs must be delimited"),
                         }
                     },
                     _ => cx.span_bug(sp, "bad thing in rhs")
@@ -239,10 +232,11 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
         ms(MatchSeq(vec!(
             ms(MatchNonterminal(lhs_nm, special_idents::matchers, 0u)),
             ms(MatchTok(FAT_ARROW)),
-            ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), Some(SEMI), false, 0u, 2u)),
+            ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), Some(SEMI),
+                                ast::OneOrMore, 0u, 2u)),
         //to phase into semicolon-termination instead of
         //semicolon-separation
-        ms(MatchSeq(vec!(ms(MatchTok(SEMI))), None, true, 2u, 2u)));
+        ms(MatchSeq(vec!(ms(MatchTok(SEMI))), None, ast::ZeroOrMore, 2u, 2u)));
 
 
     // Parse the macro_rules! invocation (`none` is for no interpolations):
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 35ec37d842a..1bb519f66cd 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use ast;
-use ast::{TokenTree, TTDelim, TTTok, TTSeq, TTNonterminal, Ident};
+use ast::{TokenTree, TtDelimited, TtToken, TtSequence, TtNonterminal, Ident};
 use codemap::{Span, DUMMY_SP};
 use diagnostic::SpanHandler;
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
@@ -18,6 +18,7 @@ use parse::token;
 use parse::lexer::TokenAndSpan;
 
 use std::rc::Rc;
+use std::ops::Add;
 use std::collections::HashMap;
 
 ///an unzipping of `TokenTree`s
@@ -44,7 +45,7 @@ pub struct TtReader<'a> {
 }
 
 /// This can do Macro-By-Example transcription. On the other hand, if
-/// `src` contains no `TTSeq`s and `TTNonterminal`s, `interp` can (and
+/// `src` contains no `TtSequence`s and `TtNonterminal`s, `interp` can (and
 /// should) be none.
 pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler,
                          interp: Option<HashMap<Ident, Rc<NamedMatch>>>,
@@ -104,37 +105,45 @@ enum LockstepIterSize {
     LisContradiction(String),
 }
 
-fn lis_merge(lhs: LockstepIterSize, rhs: LockstepIterSize) -> LockstepIterSize {
-    match lhs {
-        LisUnconstrained => rhs.clone(),
-        LisContradiction(_) => lhs.clone(),
-        LisConstraint(l_len, l_id) => match rhs {
-            LisUnconstrained => lhs.clone(),
-            LisContradiction(_) => rhs.clone(),
-            LisConstraint(r_len, _) if l_len == r_len => lhs.clone(),
-            LisConstraint(r_len, r_id) => {
-                let l_n = token::get_ident(l_id);
-                let r_n = token::get_ident(r_id);
-                LisContradiction(format!("inconsistent lockstep iteration: \
-                                          '{}' has {} items, but '{}' has {}",
-                                          l_n, l_len, r_n, r_len).to_string())
-            }
+impl Add<LockstepIterSize, LockstepIterSize> for LockstepIterSize {
+    fn add(&self, other: &LockstepIterSize) -> LockstepIterSize {
+        match *self {
+            LisUnconstrained => other.clone(),
+            LisContradiction(_) => self.clone(),
+            LisConstraint(l_len, l_id) => match *other {
+                LisUnconstrained => self.clone(),
+                LisContradiction(_) => other.clone(),
+                LisConstraint(r_len, _) if l_len == r_len => self.clone(),
+                LisConstraint(r_len, r_id) => {
+                    let l_n = token::get_ident(l_id);
+                    let r_n = token::get_ident(r_id);
+                    LisContradiction(format!("inconsistent lockstep iteration: \
+                                              '{}' has {} items, but '{}' has {}",
+                                              l_n, l_len, r_n, r_len).to_string())
+                }
+            },
         }
     }
 }
 
 fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
     match *t {
-        TTDelim(ref tts) | TTSeq(_, ref tts, _, _) => {
-            tts.iter().fold(LisUnconstrained, |lis, tt| {
-                lis_merge(lis, lockstep_iter_size(tt, r))
+        TtDelimited(_, ref delimed) => {
+            let (_, ref tts, _) = **delimed;
+            tts.iter().fold(LisUnconstrained, |size, tt| {
+                size + lockstep_iter_size(tt, r)
             })
-        }
-        TTTok(..) => LisUnconstrained,
-        TTNonterminal(_, name) => match *lookup_cur_matched(r, name) {
+        },
+        TtSequence(_, ref tts, _, _) => {
+            tts.iter().fold(LisUnconstrained, |size, tt| {
+                size + lockstep_iter_size(tt, r)
+            })
+        },
+        TtToken(..) => LisUnconstrained,
+        TtNonterminal(_, name) => match *lookup_cur_matched(r, name) {
             MatchedNonterminal(_) => LisUnconstrained,
             MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name)
-        }
+        },
     }
 }
 
@@ -189,32 +198,38 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
             }
         }
     }
-    loop { /* because it's easiest, this handles `TTDelim` not starting
-              with a `TTTok`, even though it won't happen */
+    loop { /* because it's easiest, this handles `TtDelimited` not starting
+              with a `TtToken`, even though it won't happen */
         let t = {
             let frame = r.stack.last().unwrap();
             // FIXME(pcwalton): Bad copy.
             (*frame.forest)[frame.idx].clone()
         };
         match t {
-            TTDelim(tts) => {
+            TtDelimited(_, ref delimed) => {
+                let (ref open, ref tts, ref close) = **delimed;
+                let mut forest = Vec::with_capacity(1 + tts.len() + 1);
+                forest.push(open.to_tt());
+                forest.extend(tts.iter().map(|x| (*x).clone()));
+                forest.push(close.to_tt());
+
                 r.stack.push(TtFrame {
-                    forest: tts,
+                    forest: Rc::new(forest),
                     idx: 0,
                     dotdotdoted: false,
                     sep: None
                 });
                 // if this could be 0-length, we'd need to potentially recur here
             }
-            TTTok(sp, tok) => {
+            TtToken(sp, tok) => {
                 r.cur_span = sp;
                 r.cur_tok = tok;
                 r.stack.last_mut().unwrap().idx += 1;
                 return ret_val;
             }
-            TTSeq(sp, tts, sep, zerok) => {
+            TtSequence(sp, tts, sep, kleene_op) => {
                 // FIXME(pcwalton): Bad copy.
-                match lockstep_iter_size(&TTSeq(sp, tts.clone(), sep.clone(), zerok), r) {
+                match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), kleene_op), r) {
                     LisUnconstrained => {
                         r.sp_diag.span_fatal(
                             sp.clone(), /* blame macro writer */
@@ -228,7 +243,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
                         }
                     LisConstraint(len, _) => {
                         if len == 0 {
-                            if !zerok {
+                            if kleene_op == ast::OneOrMore {
                                 // FIXME #2887 blame invoker
                                 r.sp_diag.span_fatal(sp.clone(),
                                                      "this must repeat at least once");
@@ -249,7 +264,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
                 }
             }
             // FIXME #2887: think about span stuff here
-            TTNonterminal(sp, ident) => {
+            TtNonterminal(sp, ident) => {
                 r.stack.last_mut().unwrap().idx += 1;
                 match *lookup_cur_matched(r, ident) {
                     /* sidestep the interpolation tricks for ident because