Rollup merge of #64041 - matklad:token-stream-tt, r=petrochenkov

use TokenStream rather than &[TokenTree] for built-in macros

That way, we don't loose the jointness info

13 files changed, 72 insertions, 69 deletions

diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs
index 28f907441d8..328b307361d 100644
--- a/src/libsyntax_ext/asm.rs
+++ b/src/libsyntax_ext/asm.rs
@@ -8,13 +8,12 @@ use errors::DiagnosticBuilder;
 
 use syntax::ast;
 use syntax::ext::base::{self, *};
-use syntax::parse;
 use syntax::parse::token::{self, Token};
 use syntax::ptr::P;
 use syntax::symbol::{kw, sym, Symbol};
 use syntax::ast::AsmDialect;
 use syntax_pos::Span;
-use syntax::tokenstream;
+use syntax::tokenstream::{self, TokenStream};
 use syntax::{span_err, struct_span_err};
 
 enum State {
@@ -43,7 +42,7 @@ const OPTIONS: &[Symbol] = &[sym::volatile, sym::alignstack, sym::intel];
 
 pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
                        sp: Span,
-                       tts: &[tokenstream::TokenTree])
+                       tts: TokenStream)
                        -> Box<dyn base::MacResult + 'cx> {
     let mut inline_asm = match parse_inline_asm(cx, sp, tts) {
         Ok(Some(inline_asm)) => inline_asm,
@@ -71,20 +70,20 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
 fn parse_inline_asm<'a>(
     cx: &mut ExtCtxt<'a>,
     sp: Span,
-    tts: &[tokenstream::TokenTree],
+    tts: TokenStream,
 ) -> Result<Option<ast::InlineAsm>, DiagnosticBuilder<'a>> {
     // Split the tts before the first colon, to avoid `asm!("x": y)`  being
     // parsed as `asm!(z)` with `z = "x": y` which is type ascription.
-    let first_colon = tts.iter()
+    let first_colon = tts.trees()
         .position(|tt| {
-            match *tt {
+            match tt {
                 tokenstream::TokenTree::Token(Token { kind: token::Colon, .. }) |
                 tokenstream::TokenTree::Token(Token { kind: token::ModSep, .. }) => true,
                 _ => false,
             }
         })
         .unwrap_or(tts.len());
-    let mut p = cx.new_parser_from_tts(&tts[first_colon..]);
+    let mut p = cx.new_parser_from_tts(tts.trees().skip(first_colon).collect());
     let mut asm = kw::Invalid;
     let mut asm_str_style = None;
     let mut outputs = Vec::new();
@@ -110,7 +109,8 @@ fn parse_inline_asm<'a>(
                     ));
                 }
                 // Nested parser, stop before the first colon (see above).
-                let mut p2 = cx.new_parser_from_tts(&tts[..first_colon]);
+                let mut p2 =
+                    cx.new_parser_from_tts(tts.trees().take(first_colon).collect());
 
                 if p2.token == token::Eof {
                     let mut err =
@@ -129,12 +129,8 @@ fn parse_inline_asm<'a>(
                 // This is most likely malformed.
                 if p2.token != token::Eof {
                     let mut extra_tts = p2.parse_all_token_trees()?;
-                    extra_tts.extend(tts[first_colon..].iter().cloned());
-                    p = parse::stream_to_parser(
-                        cx.parse_sess,
-                        extra_tts.into_iter().collect(),
-                        Some("inline assembly"),
-                    );
+                    extra_tts.extend(tts.trees().skip(first_colon));
+                    p = cx.new_parser_from_tts(extra_tts.into_iter().collect());
                 }
 
                 asm = s;
diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs
index 84583d0e5ec..001996e1db7 100644
--- a/src/libsyntax_ext/assert.rs
+++ b/src/libsyntax_ext/assert.rs
@@ -13,7 +13,7 @@ use syntax_pos::{Span, DUMMY_SP};
 pub fn expand_assert<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
-    tts: &[TokenTree],
+    tts: TokenStream,
 ) -> Box<dyn MacResult + 'cx> {
     let Assert { cond_expr, custom_message } = match parse_assert(cx, sp, tts) {
         Ok(assert) => assert,
@@ -59,9 +59,9 @@ struct Assert {
 fn parse_assert<'a>(
     cx: &mut ExtCtxt<'a>,
     sp: Span,
-    tts: &[TokenTree]
+    stream: TokenStream
 ) -> Result<Assert, DiagnosticBuilder<'a>> {
-    let mut parser = cx.new_parser_from_tts(tts);
+    let mut parser = cx.new_parser_from_tts(stream);
 
     if parser.token == token::Eof {
         let mut err = cx.struct_span_err(sp, "macro requires a boolean expression as an argument");
diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs
index 21cee8ae1cb..0342e442df2 100644
--- a/src/libsyntax_ext/cfg.rs
+++ b/src/libsyntax_ext/cfg.rs
@@ -7,14 +7,14 @@ use errors::DiagnosticBuilder;
 use syntax::ast;
 use syntax::ext::base::{self, *};
 use syntax::attr;
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 use syntax::parse::token;
 use syntax_pos::Span;
 
 pub fn expand_cfg(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
-    tts: &[tokenstream::TokenTree],
+    tts: TokenStream,
 ) -> Box<dyn base::MacResult + 'static> {
     let sp = cx.with_legacy_ctxt(sp);
 
@@ -33,7 +33,7 @@ pub fn expand_cfg(
 fn parse_cfg<'a>(
     cx: &mut ExtCtxt<'a>,
     sp: Span,
-    tts: &[tokenstream::TokenTree],
+    tts: TokenStream,
 ) -> Result<ast::MetaItem, DiagnosticBuilder<'a>> {
     let mut p = cx.new_parser_from_tts(tts);
 
diff --git a/src/libsyntax_ext/compile_error.rs b/src/libsyntax_ext/compile_error.rs
index 59d3f2c9c78..24f3a66d4ae 100644
--- a/src/libsyntax_ext/compile_error.rs
+++ b/src/libsyntax_ext/compile_error.rs
@@ -2,11 +2,11 @@
 
 use syntax::ext::base::{self, *};
 use syntax_pos::Span;
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 
 pub fn expand_compile_error<'cx>(cx: &'cx mut ExtCtxt<'_>,
                               sp: Span,
-                              tts: &[tokenstream::TokenTree])
+                              tts: TokenStream)
                               -> Box<dyn base::MacResult + 'cx> {
     let var = match get_single_str_from_tts(cx, sp, tts, "compile_error!") {
         None => return DummyResult::any(sp),
diff --git a/src/libsyntax_ext/concat.rs b/src/libsyntax_ext/concat.rs
index ffa5154ca0c..fc56dff65e4 100644
--- a/src/libsyntax_ext/concat.rs
+++ b/src/libsyntax_ext/concat.rs
@@ -1,14 +1,14 @@
 use syntax::ast;
 use syntax::ext::base::{self, DummyResult};
 use syntax::symbol::Symbol;
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 
 use std::string::String;
 
-pub fn expand_syntax_ext(
+pub fn expand_concat(
     cx: &mut base::ExtCtxt<'_>,
     sp: syntax_pos::Span,
-    tts: &[tokenstream::TokenTree],
+    tts: TokenStream,
 ) -> Box<dyn base::MacResult + 'static> {
     let es = match base::get_exprs_from_tts(cx, sp, tts) {
         Some(e) => e,
diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs
index 96677072d1b..6391b62b58d 100644
--- a/src/libsyntax_ext/concat_idents.rs
+++ b/src/libsyntax_ext/concat_idents.rs
@@ -6,21 +6,21 @@ use syntax::parse::token::{self, Token};
 use syntax::ptr::P;
 use syntax_pos::Span;
 use syntax_pos::symbol::Symbol;
-use syntax::tokenstream::TokenTree;
+use syntax::tokenstream::{TokenTree, TokenStream};
 
-pub fn expand_syntax_ext<'cx>(cx: &'cx mut ExtCtxt<'_>,
-                              sp: Span,
-                              tts: &[TokenTree])
-                              -> Box<dyn base::MacResult + 'cx> {
+pub fn expand_concat_idents<'cx>(cx: &'cx mut ExtCtxt<'_>,
+                                 sp: Span,
+                                 tts: TokenStream)
+                                 -> Box<dyn base::MacResult + 'cx> {
     if tts.is_empty() {
         cx.span_err(sp, "concat_idents! takes 1 or more arguments.");
         return DummyResult::any(sp);
     }
 
     let mut res_str = String::new();
-    for (i, e) in tts.iter().enumerate() {
+    for (i, e) in tts.into_trees().enumerate() {
         if i & 1 == 1 {
-            match *e {
+            match e {
                 TokenTree::Token(Token { kind: token::Comma, .. }) => {}
                 _ => {
                     cx.span_err(sp, "concat_idents! expecting comma.");
@@ -28,7 +28,7 @@ pub fn expand_syntax_ext<'cx>(cx: &'cx mut ExtCtxt<'_>,
                 }
             }
         } else {
-            match *e {
+            match e {
                 TokenTree::Token(Token { kind: token::Ident(name, _), .. }) =>
                     res_str.push_str(&name.as_str()),
                 _ => {
diff --git a/src/libsyntax_ext/env.rs b/src/libsyntax_ext/env.rs
index 6343d218de8..179b7fe00a9 100644
--- a/src/libsyntax_ext/env.rs
+++ b/src/libsyntax_ext/env.rs
@@ -7,13 +7,13 @@ use syntax::ast::{self, Ident, GenericArg};
 use syntax::ext::base::{self, *};
 use syntax::symbol::{kw, sym, Symbol};
 use syntax_pos::Span;
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 
 use std::env;
 
 pub fn expand_option_env<'cx>(cx: &'cx mut ExtCtxt<'_>,
                               sp: Span,
-                              tts: &[tokenstream::TokenTree])
+                              tts: TokenStream)
                               -> Box<dyn base::MacResult + 'cx> {
     let var = match get_single_str_from_tts(cx, sp, tts, "option_env!") {
         None => return DummyResult::any(sp),
@@ -45,7 +45,7 @@ pub fn expand_option_env<'cx>(cx: &'cx mut ExtCtxt<'_>,
 
 pub fn expand_env<'cx>(cx: &'cx mut ExtCtxt<'_>,
                        sp: Span,
-                       tts: &[tokenstream::TokenTree])
+                       tts: TokenStream)
                        -> Box<dyn base::MacResult + 'cx> {
     let mut exprs = match get_exprs_from_tts(cx, sp, tts) {
         Some(ref exprs) if exprs.is_empty() => {
diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs
index 47394c02b41..ad275f421af 100644
--- a/src/libsyntax_ext/format.rs
+++ b/src/libsyntax_ext/format.rs
@@ -11,7 +11,7 @@ use syntax::ext::base::{self, *};
 use syntax::parse::token;
 use syntax::ptr::P;
 use syntax::symbol::{Symbol, sym};
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 use syntax_pos::{MultiSpan, Span};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
@@ -126,7 +126,7 @@ struct Context<'a, 'b> {
 fn parse_args<'a>(
     ecx: &mut ExtCtxt<'a>,
     sp: Span,
-    tts: &[tokenstream::TokenTree]
+    tts: TokenStream,
 ) -> Result<(P<ast::Expr>, Vec<P<ast::Expr>>, FxHashMap<Symbol, usize>), DiagnosticBuilder<'a>> {
     let mut args = Vec::<P<ast::Expr>>::new();
     let mut names = FxHashMap::<Symbol, usize>::default();
@@ -794,7 +794,7 @@ impl<'a, 'b> Context<'a, 'b> {
 fn expand_format_args_impl<'cx>(
     ecx: &'cx mut ExtCtxt<'_>,
     mut sp: Span,
-    tts: &[tokenstream::TokenTree],
+    tts: TokenStream,
     nl: bool,
 ) -> Box<dyn base::MacResult + 'cx> {
     sp = ecx.with_def_site_ctxt(sp);
@@ -812,7 +812,7 @@ fn expand_format_args_impl<'cx>(
 pub fn expand_format_args<'cx>(
     ecx: &'cx mut ExtCtxt<'_>,
     sp: Span,
-    tts: &[tokenstream::TokenTree],
+    tts: TokenStream,
 ) -> Box<dyn base::MacResult + 'cx> {
     expand_format_args_impl(ecx, sp, tts, false)
 }
@@ -820,7 +820,7 @@ pub fn expand_format_args<'cx>(
 pub fn expand_format_args_nl<'cx>(
     ecx: &'cx mut ExtCtxt<'_>,
     sp: Span,
-    tts: &[tokenstream::TokenTree],
+    tts: TokenStream,
 ) -> Box<dyn base::MacResult + 'cx> {
     expand_format_args_impl(ecx, sp, tts, true)
 }
diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs
index a8b61593db7..6140f0df58a 100644
--- a/src/libsyntax_ext/global_asm.rs
+++ b/src/libsyntax_ext/global_asm.rs
@@ -16,12 +16,12 @@ use syntax::ext::base::{self, *};
 use syntax::parse::token;
 use syntax::ptr::P;
 use syntax_pos::Span;
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 use smallvec::smallvec;
 
 pub fn expand_global_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
                               sp: Span,
-                              tts: &[tokenstream::TokenTree]) -> Box<dyn base::MacResult + 'cx> {
+                              tts: TokenStream) -> Box<dyn base::MacResult + 'cx> {
     match parse_global_asm(cx, sp, tts) {
         Ok(Some(global_asm)) => {
             MacEager::items(smallvec![P(ast::Item {
@@ -45,7 +45,7 @@ pub fn expand_global_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
 fn parse_global_asm<'a>(
     cx: &mut ExtCtxt<'a>,
     sp: Span,
-    tts: &[tokenstream::TokenTree]
+    tts: TokenStream
 ) -> Result<Option<ast::GlobalAsm>, DiagnosticBuilder<'a>> {
     let mut p = cx.new_parser_from_tts(tts);
 
diff --git a/src/libsyntax_ext/lib.rs b/src/libsyntax_ext/lib.rs
index 1a617691662..26ef80b2b06 100644
--- a/src/libsyntax_ext/lib.rs
+++ b/src/libsyntax_ext/lib.rs
@@ -67,8 +67,8 @@ pub fn register_builtin_macros(resolver: &mut dyn syntax::ext::base::Resolver, e
         cfg: cfg::expand_cfg,
         column: source_util::expand_column,
         compile_error: compile_error::expand_compile_error,
-        concat_idents: concat_idents::expand_syntax_ext,
-        concat: concat::expand_syntax_ext,
+        concat_idents: concat_idents::expand_concat_idents,
+        concat: concat::expand_concat,
         env: env::expand_env,
         file: source_util::expand_file,
         format_args_nl: format::expand_format_args_nl,
@@ -78,7 +78,7 @@ pub fn register_builtin_macros(resolver: &mut dyn syntax::ext::base::Resolver, e
         include_str: source_util::expand_include_str,
         include: source_util::expand_include,
         line: source_util::expand_line,
-        log_syntax: log_syntax::expand_syntax_ext,
+        log_syntax: log_syntax::expand_log_syntax,
         module_path: source_util::expand_mod,
         option_env: env::expand_option_env,
         stringify: source_util::expand_stringify,
diff --git a/src/libsyntax_ext/log_syntax.rs b/src/libsyntax_ext/log_syntax.rs
index cbdfd08b497..92130bfaf68 100644
--- a/src/libsyntax_ext/log_syntax.rs
+++ b/src/libsyntax_ext/log_syntax.rs
@@ -1,11 +1,11 @@
 use syntax::ext::base;
 use syntax::print;
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 use syntax_pos;
 
-pub fn expand_syntax_ext<'cx>(_cx: &'cx mut base::ExtCtxt<'_>,
+pub fn expand_log_syntax<'cx>(_cx: &'cx mut base::ExtCtxt<'_>,
                               sp: syntax_pos::Span,
-                              tts: &[tokenstream::TokenTree])
+                              tts: TokenStream)
                               -> Box<dyn base::MacResult + 'cx> {
     println!("{}", print::pprust::tts_to_string(tts));
 
diff --git a/src/libsyntax_ext/source_util.rs b/src/libsyntax_ext/source_util.rs
index e008ed710e4..9dc9d66b86f 100644
--- a/src/libsyntax_ext/source_util.rs
+++ b/src/libsyntax_ext/source_util.rs
@@ -4,7 +4,7 @@ use syntax::parse::{self, token, DirectoryOwnership};
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::symbol::Symbol;
-use syntax::tokenstream;
+use syntax::tokenstream::TokenStream;
 
 use smallvec::SmallVec;
 use syntax_pos::{self, Pos, Span};
@@ -16,7 +16,7 @@ use rustc_data_structures::sync::Lrc;
 // a given file into the current one.
 
 /// line!(): expands to the current line number
-pub fn expand_line(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_line(cx: &mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                    -> Box<dyn base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "line!");
 
@@ -27,7 +27,7 @@ pub fn expand_line(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree
 }
 
 /* column!(): expands to the current column number */
-pub fn expand_column(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_column(cx: &mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                   -> Box<dyn base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "column!");
 
@@ -40,7 +40,7 @@ pub fn expand_column(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTr
 /// file!(): expands to the current filename */
 /// The source_file (`loc.file`) contains a bunch more information we could spit
 /// out if we wanted.
-pub fn expand_file(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_file(cx: &mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                    -> Box<dyn base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "file!");
 
@@ -49,13 +49,13 @@ pub fn expand_file(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree
     base::MacEager::expr(cx.expr_str(topmost, Symbol::intern(&loc.file.name.to_string())))
 }
 
-pub fn expand_stringify(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_stringify(cx: &mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                         -> Box<dyn base::MacResult+'static> {
     let s = pprust::tts_to_string(tts);
     base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&s)))
 }
 
-pub fn expand_mod(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_mod(cx: &mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                   -> Box<dyn base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "module_path!");
     let mod_path = &cx.current_expansion.module.mod_path;
@@ -67,7 +67,7 @@ pub fn expand_mod(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]
 /// include! : parse the given file as an expr
 /// This is generally a bad idea because it's going to behave
 /// unhygienically.
-pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                            -> Box<dyn base::MacResult+'cx> {
     let file = match get_single_str_from_tts(cx, sp, tts, "include!") {
         Some(f) => f,
@@ -105,7 +105,7 @@ pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt<'_>, sp: Span, tts: &[tokenstrea
 }
 
 // include_str! : read the given file, insert it as a literal string expr
-pub fn expand_include_str(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_include_str(cx: &mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                           -> Box<dyn base::MacResult+'static> {
     let file = match get_single_str_from_tts(cx, sp, tts, "include_str!") {
         Some(f) => f,
@@ -130,7 +130,7 @@ pub fn expand_include_str(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::To
     }
 }
 
-pub fn expand_include_bytes(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree])
+pub fn expand_include_bytes(cx: &mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
                             -> Box<dyn base::MacResult+'static> {
     let file = match get_single_str_from_tts(cx, sp, tts, "include_bytes!") {
         Some(f) => f,
diff --git a/src/libsyntax_ext/trace_macros.rs b/src/libsyntax_ext/trace_macros.rs
index 0dce8a36f4c..d83c24046d9 100644
--- a/src/libsyntax_ext/trace_macros.rs
+++ b/src/libsyntax_ext/trace_macros.rs
@@ -1,20 +1,27 @@
 use syntax::ext::base::{self, ExtCtxt};
 use syntax::symbol::kw;
 use syntax_pos::Span;
-use syntax::tokenstream::TokenTree;
+use syntax::tokenstream::{TokenTree, TokenStream};
 
 pub fn expand_trace_macros(cx: &mut ExtCtxt<'_>,
                            sp: Span,
-                           tt: &[TokenTree])
+                           tt: TokenStream)
                            -> Box<dyn base::MacResult + 'static> {
-    match tt {
-        [TokenTree::Token(token)] if token.is_keyword(kw::True) => {
-            cx.set_trace_macros(true);
-        }
-        [TokenTree::Token(token)] if token.is_keyword(kw::False) => {
-            cx.set_trace_macros(false);
-        }
-        _ => cx.span_err(sp, "trace_macros! accepts only `true` or `false`"),
+    let mut cursor = tt.into_trees();
+    let mut err = false;
+    let value = match &cursor.next() {
+        Some(TokenTree::Token(token)) if token.is_keyword(kw::True) => true,
+        Some(TokenTree::Token(token)) if token.is_keyword(kw::False) => false,
+        _ => {
+            err = true;
+            false
+        },
+    };
+    err |= cursor.next().is_some();
+    if err {
+        cx.span_err(sp, "trace_macros! accepts only `true` or `false`")
+    } else {
+        cx.set_trace_macros(value);
     }
 
     base::DummyResult::any_valid(sp)