Fix partially consumed tokens in macro matchers.

2 files changed, 14 insertions, 12 deletions

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index b0696a986e3..d7a5d04bda5 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -476,24 +476,21 @@ pub fn parse(sess: &ParseSess,
                 }
                 rdr.next_token();
             } else /* bb_eis.len() == 1 */ {
-                let mut rust_parser = Parser::new(sess, cfg.clone(), Box::new(rdr.clone()));
-
-                let mut ei = bb_eis.pop().unwrap();
-                match ei.top_elts.get_tt(ei.idx) {
-                    TokenTree::Token(span, MatchNt(_, ident)) => {
+                rdr.next_tok = {
+                    let mut rust_parser = Parser::new(sess, cfg.clone(), Box::new(&mut rdr));
+                    let mut ei = bb_eis.pop().unwrap();
+                    if let TokenTree::Token(span, MatchNt(_, ident)) = ei.top_elts.get_tt(ei.idx) {
                         let match_cur = ei.match_cur;
                         (&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal(
                             parse_nt(&mut rust_parser, span, &ident.name.as_str()))));
                         ei.idx += 1;
                         ei.match_cur += 1;
+                    } else {
+                        unreachable!()
                     }
-                    _ => panic!()
-                }
-                cur_eis.push(ei);
-
-                for _ in 0..rust_parser.tokens_consumed {
-                    let _ = rdr.next_token();
-                }
+                    cur_eis.push(ei);
+                    Some(TokenAndSpan { tok: rust_parser.token, sp: rust_parser.span })
+                };
             }
         }
 
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 939425378de..205c709d6cb 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -48,6 +48,7 @@ pub struct TtReader<'a> {
     /* cached: */
     pub cur_tok: Token,
     pub cur_span: Span,
+    pub next_tok: Option<TokenAndSpan>,
     /// Transform doc comments. Only useful in macro invocations
     pub desugar_doc_comments: bool,
     pub fatal_errs: Vec<DiagnosticBuilder<'a>>,
@@ -100,6 +101,7 @@ pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler,
         /* dummy values, never read: */
         cur_tok: token::Eof,
         cur_span: DUMMY_SP,
+        next_tok: None,
         fatal_errs: Vec::new(),
     };
     tt_next_token(&mut r); /* get cur_tok and cur_span set up */
@@ -178,6 +180,9 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
 /// Return the next token from the TtReader.
 /// EFFECT: advances the reader's token field
 pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
+    if let Some(tok) = r.next_tok.take() {
+        return tok;
+    }
     // FIXME(pcwalton): Bad copy?
     let ret_val = TokenAndSpan {
         tok: r.cur_tok.clone(),