Auto merge of #34908 - jseyfried:improve_tt_matchers, r=nrc

macros: Improve `tt` matchers

Fixes #5846, fixes #22819.
r? @nrc

2 files changed, 17 insertions, 3 deletions

diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 7c0d10669f3..29a300b172e 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -14,7 +14,7 @@ use syntax_pos::{Span, DUMMY_SP};
 use errors::{Handler, DiagnosticBuilder};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
 use parse::token::{DocComment, MatchNt, SubstNt};
-use parse::token::{Token, NtIdent, SpecialMacroVar};
+use parse::token::{Token, Interpolated, NtIdent, NtTT, SpecialMacroVar};
 use parse::token;
 use parse::lexer::TokenAndSpan;
 use tokenstream::{self, TokenTree};
@@ -278,9 +278,9 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
             }
             // FIXME #2887: think about span stuff here
             TokenTree::Token(sp, SubstNt(ident)) => {
-                r.stack.last_mut().unwrap().idx += 1;
                 match lookup_cur_matched(r, ident) {
                     None => {
+                        r.stack.last_mut().unwrap().idx += 1;
                         r.cur_span = sp;
                         r.cur_tok = SubstNt(ident);
                         return ret_val;
@@ -292,14 +292,24 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
                             // (a) idents can be in lots of places, so it'd be a pain
                             // (b) we actually can, since it's a token.
                             MatchedNonterminal(NtIdent(ref sn)) => {
+                                r.stack.last_mut().unwrap().idx += 1;
                                 r.cur_span = sn.span;
                                 r.cur_tok = token::Ident(sn.node);
                                 return ret_val;
                             }
+                            MatchedNonterminal(NtTT(ref tt)) => {
+                                r.stack.push(TtFrame {
+                                    forest: TokenTree::Token(sp, Interpolated(NtTT(tt.clone()))),
+                                    idx: 0,
+                                    dotdotdoted: false,
+                                    sep: None,
+                                });
+                            }
                             MatchedNonterminal(ref other_whole_nt) => {
+                                r.stack.last_mut().unwrap().idx += 1;
                                 // FIXME(pcwalton): Bad copy.
                                 r.cur_span = sp;
-                                r.cur_tok = token::Interpolated((*other_whole_nt).clone());
+                                r.cur_tok = Interpolated((*other_whole_nt).clone());
                                 return ret_val;
                             }
                             MatchedSeq(..) => {
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index 0ad09fd0f7d..d38edf81688 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -135,6 +135,7 @@ impl TokenTree {
             }
             TokenTree::Token(_, token::SpecialVarNt(..)) => 2,
             TokenTree::Token(_, token::MatchNt(..)) => 3,
+            TokenTree::Token(_, token::Interpolated(Nonterminal::NtTT(..))) => 1,
             TokenTree::Delimited(_, ref delimed) => delimed.tts.len() + 2,
             TokenTree::Sequence(_, ref seq) => seq.tts.len(),
             TokenTree::Token(..) => 0,
@@ -197,6 +198,9 @@ impl TokenTree {
                          TokenTree::Token(sp, token::Ident(kind))];
                 v[index].clone()
             }
+            (&TokenTree::Token(_, token::Interpolated(Nonterminal::NtTT(ref tt))), _) => {
+                tt.clone().unwrap()
+            }
             (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(),
             _ => panic!("Cannot expand a token tree"),
         }