Rollup merge of #34385 - cgswords:tstream, r=nrc

syntax-[breaking-change] cc #31645
(Only breaking because ast::TokenTree is now tokenstream::TokenTree.)

This pull request refactors TokenTrees into their own file as src/libsyntax/tokenstream.rs, moving them out of src/libsyntax/ast.rs, in order to prepare for an accompanying TokenStream implementation (per RFC 1566).

8 files changed, 17 insertions, 10 deletions

diff --git a/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs b/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs
index 5fdca58dec2..a6bc9db199c 100644
--- a/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs
+++ b/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs
@@ -17,10 +17,11 @@ extern crate syntax_pos;
 extern crate rustc;
 extern crate rustc_plugin;
 
-use syntax::ast::{self, TokenTree, Item, MetaItem, ImplItem, TraitItem, ItemKind};
+use syntax::ast::{self, Item, MetaItem, ImplItem, TraitItem, ItemKind};
 use syntax::ext::base::*;
 use syntax::parse::{self, token};
 use syntax::ptr::P;
+use syntax::tokenstream::TokenTree;
 use syntax_pos::Span;
 use rustc_plugin::Registry;
 
diff --git a/src/test/run-pass-fulldeps/auxiliary/issue_16723_multiple_items_syntax_ext.rs b/src/test/run-pass-fulldeps/auxiliary/issue_16723_multiple_items_syntax_ext.rs
index 9f0c8d4d649..7f8a741465b 100644
--- a/src/test/run-pass-fulldeps/auxiliary/issue_16723_multiple_items_syntax_ext.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/issue_16723_multiple_items_syntax_ext.rs
@@ -21,6 +21,7 @@ extern crate syntax_pos;
 use syntax::ast;
 use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
 use syntax::util::small_vector::SmallVector;
+use syntax::tokenstream;
 use rustc_plugin::Registry;
 
 #[plugin_registrar]
@@ -28,7 +29,8 @@ pub fn plugin_registrar(reg: &mut Registry) {
     reg.register_macro("multiple_items", expand)
 }
 
-fn expand(cx: &mut ExtCtxt, _: syntax_pos::Span, _: &[ast::TokenTree]) -> Box<MacResult+'static> {
+fn expand(cx: &mut ExtCtxt, _: syntax_pos::Span, _: &[tokenstream::TokenTree])
+          -> Box<MacResult+'static> {
     MacEager::items(SmallVector::many(vec![
         quote_item!(cx, struct Struct1;).unwrap(),
         quote_item!(cx, struct Struct2;).unwrap()
diff --git a/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs b/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs
index e32c83472a1..11d81eda556 100644
--- a/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs
@@ -17,10 +17,11 @@ extern crate rustc;
 extern crate rustc_plugin;
 extern crate syntax_pos;
 
-use syntax::ast::{self, TokenTree, Item, MetaItem, ImplItem, TraitItem, ItemKind};
+use syntax::ast::{self, Item, MetaItem, ImplItem, TraitItem, ItemKind};
 use syntax::ext::base::*;
 use syntax::parse::{self, token};
 use syntax::ptr::P;
+use syntax::tokenstream::TokenTree;
 use syntax_pos::Span;
 use rustc_plugin::Registry;
 
diff --git a/src/test/run-pass-fulldeps/auxiliary/plugin_args.rs b/src/test/run-pass-fulldeps/auxiliary/plugin_args.rs
index e47e38bb318..f0edc0f2b12 100644
--- a/src/test/run-pass-fulldeps/auxiliary/plugin_args.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/plugin_args.rs
@@ -26,6 +26,7 @@ use syntax::parse::token;
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax_pos::Span;
+use syntax::tokenstream;
 use rustc_plugin::Registry;
 
 struct Expander {
@@ -36,7 +37,7 @@ impl TTMacroExpander for Expander {
     fn expand<'cx>(&self,
                    ecx: &'cx mut ExtCtxt,
                    sp: Span,
-                   _: &[ast::TokenTree]) -> Box<MacResult+'cx> {
+                   _: &[tokenstream::TokenTree]) -> Box<MacResult+'cx> {
         let args = self.args.iter().map(|i| pprust::meta_item_to_string(&*i))
             .collect::<Vec<_>>().join(", ");
         let interned = token::intern_and_get_ident(&args[..]);
diff --git a/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs b/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs
index 5566692a4e2..5b1e210b0b2 100644
--- a/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs
@@ -19,7 +19,8 @@ extern crate rustc;
 extern crate rustc_plugin;
 
 use syntax::parse::token::{self, str_to_ident, NtExpr, NtPat};
-use syntax::ast::{TokenTree, Pat};
+use syntax::ast::{Pat};
+use syntax::tokenstream::{TokenTree};
 use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
 use syntax::ext::build::AstBuilder;
 use syntax::ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal};
diff --git a/src/test/run-pass-fulldeps/auxiliary/roman_numerals.rs b/src/test/run-pass-fulldeps/auxiliary/roman_numerals.rs
index 7a3729015c1..0c8af013fd1 100644
--- a/src/test/run-pass-fulldeps/auxiliary/roman_numerals.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/roman_numerals.rs
@@ -19,8 +19,8 @@ extern crate syntax_pos;
 extern crate rustc;
 extern crate rustc_plugin;
 
-use syntax::ast::TokenTree;
 use syntax::parse::token;
+use syntax::tokenstream::TokenTree;
 use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
 use syntax::ext::build::AstBuilder;  // trait for expr_usize
 use syntax_pos::Span;
diff --git a/src/test/run-pass-fulldeps/auxiliary/syntax_extension_with_dll_deps_2.rs b/src/test/run-pass-fulldeps/auxiliary/syntax_extension_with_dll_deps_2.rs
index 7581e43f12e..72d26285355 100644
--- a/src/test/run-pass-fulldeps/auxiliary/syntax_extension_with_dll_deps_2.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/syntax_extension_with_dll_deps_2.rs
@@ -19,8 +19,9 @@ extern crate syntax_pos;
 extern crate rustc;
 extern crate rustc_plugin;
 
-use syntax::ast::{TokenTree, Item, MetaItem};
+use syntax::ast::{Item, MetaItem};
 use syntax::ext::base::*;
+use syntax::tokenstream::TokenTree;
 use syntax_pos::Span;
 use rustc_plugin::Registry;
 
diff --git a/src/test/run-pass-fulldeps/quote-tokens.rs b/src/test/run-pass-fulldeps/quote-tokens.rs
index 4397da35d7a..710e2fd1d07 100644
--- a/src/test/run-pass-fulldeps/quote-tokens.rs
+++ b/src/test/run-pass-fulldeps/quote-tokens.rs
@@ -20,8 +20,8 @@ use syntax::ptr::P;
 use syntax::parse::PResult;
 
 fn syntax_extension(cx: &ExtCtxt) {
-    let e_toks : Vec<syntax::ast::TokenTree> = quote_tokens!(cx, 1 + 2);
-    let p_toks : Vec<syntax::ast::TokenTree> = quote_tokens!(cx, (x, 1 .. 4, *));
+    let e_toks : Vec<syntax::tokenstream::TokenTree> = quote_tokens!(cx, 1 + 2);
+    let p_toks : Vec<syntax::tokenstream::TokenTree> = quote_tokens!(cx, (x, 1 .. 4, *));
 
     let a: P<syntax::ast::Expr> = quote_expr!(cx, 1 + 2);
     let _b: Option<P<syntax::ast::Item>> = quote_item!(cx, static foo : isize = $e_toks; );
@@ -39,7 +39,7 @@ fn syntax_extension(cx: &ExtCtxt) {
 
     let _l: P<syntax::ast::Ty> = quote_ty!(cx, &isize);
 
-    let _m: Vec<syntax::ast::TokenTree> = quote_matcher!(cx, $($foo:tt,)* bar);
+    let _m: Vec<syntax::tokenstream::TokenTree> = quote_matcher!(cx, $($foo:tt,)* bar);
     let _n: syntax::ast::Attribute = quote_attr!(cx, #![cfg(foo, bar = "baz")]);
 
     let _o: Option<P<syntax::ast::Item>> = quote_item!(cx, fn foo<T: ?Sized>() {});