Rollup merge of #34385 - cgswords:tstream, r=nrc

syntax-[breaking-change] cc #31645
(Only breaking because ast::TokenTree is now tokenstream::TokenTree.)

This pull request refactors TokenTrees into their own file as src/libsyntax/tokenstream.rs, moving them out of src/libsyntax/ast.rs, in order to prepare for an accompanying TokenStream implementation (per RFC 1566).

8 files changed, 19 insertions, 14 deletions

diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs
index a8f5171f800..e9e72c040fe 100644
--- a/src/libsyntax_ext/asm.rs
+++ b/src/libsyntax_ext/asm.rs
@@ -23,6 +23,7 @@ use syntax::parse::{self, token};
 use syntax::ptr::P;
 use syntax::ast::AsmDialect;
 use syntax_pos::Span;
+use syntax::tokenstream;
 
 enum State {
     Asm,
@@ -48,7 +49,7 @@ impl State {
 
 const OPTIONS: &'static [&'static str] = &["volatile", "alignstack", "intel"];
 
-pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                        -> Box<base::MacResult+'cx> {
     if !cx.ecfg.enable_asm() {
         feature_gate::emit_feature_err(
@@ -62,8 +63,8 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
     // parsed as `asm!(z)` with `z = "x": y` which is type ascription.
     let first_colon = tts.iter().position(|tt| {
         match *tt {
-            ast::TokenTree::Token(_, token::Colon) |
-            ast::TokenTree::Token(_, token::ModSep) => true,
+            tokenstream::TokenTree::Token(_, token::Colon) |
+            tokenstream::TokenTree::Token(_, token::ModSep) => true,
             _ => false
         }
     }).unwrap_or(tts.len());
diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs
index 201c9f38978..dbf23328f41 100644
--- a/src/libsyntax_ext/cfg.rs
+++ b/src/libsyntax_ext/cfg.rs
@@ -12,17 +12,17 @@
 /// a literal `true` or `false` based on whether the given cfg matches the
 /// current compilation environment.
 
-use syntax::ast;
 use syntax::ext::base::*;
 use syntax::ext::base;
 use syntax::ext::build::AstBuilder;
 use syntax::attr;
+use syntax::tokenstream;
 use syntax::parse::token;
 use syntax_pos::Span;
 
 pub fn expand_cfg<'cx>(cx: &mut ExtCtxt,
                        sp: Span,
-                       tts: &[ast::TokenTree])
+                       tts: &[tokenstream::TokenTree])
                        -> Box<base::MacResult+'static> {
     let mut p = cx.new_parser_from_tts(tts);
     let cfg = panictry!(p.parse_meta_item());
diff --git a/src/libsyntax_ext/concat.rs b/src/libsyntax_ext/concat.rs
index c754cb14b67..22c4aeefbd1 100644
--- a/src/libsyntax_ext/concat.rs
+++ b/src/libsyntax_ext/concat.rs
@@ -13,12 +13,13 @@ use syntax::ext::base;
 use syntax::ext::build::AstBuilder;
 use syntax::parse::token;
 use syntax_pos;
+use syntax::tokenstream;
 
 use std::string::String;
 
 pub fn expand_syntax_ext(cx: &mut base::ExtCtxt,
                          sp: syntax_pos::Span,
-                         tts: &[ast::TokenTree])
+                         tts: &[tokenstream::TokenTree])
                          -> Box<base::MacResult+'static> {
     let es = match base::get_exprs_from_tts(cx, sp, tts) {
         Some(e) => e,
diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs
index 5a98925ba82..2be32d33345 100644
--- a/src/libsyntax_ext/concat_idents.rs
+++ b/src/libsyntax_ext/concat_idents.rs
@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use syntax::ast::{self, TokenTree};
+use syntax::ast;
 use syntax::ext::base::*;
 use syntax::ext::base;
 use syntax::feature_gate;
@@ -16,6 +16,7 @@ use syntax::parse::token;
 use syntax::parse::token::str_to_ident;
 use syntax::ptr::P;
 use syntax_pos::Span;
+use syntax::tokenstream::TokenTree;
 
 pub fn expand_syntax_ext<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree])
                               -> Box<base::MacResult+'cx> {
diff --git a/src/libsyntax_ext/env.rs b/src/libsyntax_ext/env.rs
index de18153d962..546f8eaa692 100644
--- a/src/libsyntax_ext/env.rs
+++ b/src/libsyntax_ext/env.rs
@@ -20,10 +20,11 @@ use syntax::ext::base;
 use syntax::ext::build::AstBuilder;
 use syntax::parse::token;
 use syntax_pos::Span;
+use syntax::tokenstream;
 
 use std::env;
 
-pub fn expand_option_env<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_option_env<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                               -> Box<base::MacResult+'cx> {
     let var = match get_single_str_from_tts(cx, sp, tts, "option_env!") {
         None => return DummyResult::expr(sp),
@@ -56,7 +57,7 @@ pub fn expand_option_env<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenT
     MacEager::expr(e)
 }
 
-pub fn expand_env<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_env<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                        -> Box<base::MacResult+'cx> {
     let mut exprs = match get_exprs_from_tts(cx, sp, tts) {
         Some(ref exprs) if exprs.is_empty() => {
diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs
index f928e577d31..a1454452160 100644
--- a/src/libsyntax_ext/format.rs
+++ b/src/libsyntax_ext/format.rs
@@ -22,6 +22,7 @@ use syntax::fold::Folder;
 use syntax::parse::token::{self, keywords};
 use syntax::ptr::P;
 use syntax_pos::{Span, DUMMY_SP};
+use syntax::tokenstream;
 
 use std::collections::HashMap;
 
@@ -81,7 +82,7 @@ struct Context<'a, 'b:'a> {
 /// Some((fmtstr, unnamed arguments, ordering of named arguments,
 ///       named arguments))
 /// ```
-fn parse_args(ecx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+fn parse_args(ecx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
               -> Option<(P<ast::Expr>, Vec<P<ast::Expr>>, Vec<String>,
                          HashMap<String, P<ast::Expr>>)> {
     let mut args = Vec::new();
@@ -607,7 +608,7 @@ impl<'a, 'b> Context<'a, 'b> {
 }
 
 pub fn expand_format_args<'cx>(ecx: &'cx mut ExtCtxt, sp: Span,
-                               tts: &[ast::TokenTree])
+                               tts: &[tokenstream::TokenTree])
                                -> Box<base::MacResult+'cx> {
 
     match parse_args(ecx, sp, tts) {
diff --git a/src/libsyntax_ext/log_syntax.rs b/src/libsyntax_ext/log_syntax.rs
index a84dc9d619f..9645c5bb427 100644
--- a/src/libsyntax_ext/log_syntax.rs
+++ b/src/libsyntax_ext/log_syntax.rs
@@ -8,15 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use syntax::ast;
 use syntax::ext::base;
 use syntax::feature_gate;
 use syntax::print;
+use syntax::tokenstream;
 use syntax_pos;
 
 pub fn expand_syntax_ext<'cx>(cx: &'cx mut base::ExtCtxt,
                               sp: syntax_pos::Span,
-                              tts: &[ast::TokenTree])
+                              tts: &[tokenstream::TokenTree])
                               -> Box<base::MacResult+'cx> {
     if !cx.ecfg.enable_log_syntax() {
         feature_gate::emit_feature_err(&cx.parse_sess.span_diagnostic,
diff --git a/src/libsyntax_ext/trace_macros.rs b/src/libsyntax_ext/trace_macros.rs
index 13892d53092..ad396d38de9 100644
--- a/src/libsyntax_ext/trace_macros.rs
+++ b/src/libsyntax_ext/trace_macros.rs
@@ -8,12 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use syntax::ast::TokenTree;
 use syntax::ext::base::ExtCtxt;
 use syntax::ext::base;
 use syntax::feature_gate;
 use syntax::parse::token::keywords;
 use syntax_pos::Span;
+use syntax::tokenstream::TokenTree;
 
 pub fn expand_trace_macros(cx: &mut ExtCtxt,
                            sp: Span,