Auto merge of #56369 - nnethercote:rm-Delimited, r=petrochenkov

Remove `tokenstream::Delimited`.

Because it's an extra type layer that doesn't really help; in a couple
of places it actively gets in the way, and overall removing it makes the
code nicer. It does, however, move `tokenstream::TokenTree` further away
from the `TokenTree` in `quote.rs`.

More importantly, this change reduces the size of `TokenStream` from 48
bytes to 40 bytes on x86-64, which is enough to slightly reduce
instruction counts on numerous benchmarks, the best by 1.5%.

Note that `open_tt` and `close_tt` have gone from being methods on
`Delimited` to associated methods of `TokenTree`.

4 files changed, 31 insertions, 32 deletions

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 06e7b465784..f4e9a7e4093 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -622,9 +622,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
         let mut trees = tokens.trees();
         match trees.next() {
-            Some(TokenTree::Delimited(_, delim)) => {
+            Some(TokenTree::Delimited(_, _, tts)) => {
                 if trees.next().is_none() {
-                    return delim.tts.into()
+                    return tts.into()
                 }
             }
             Some(TokenTree::Token(..)) => {}
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 91818992fe1..c3497a17806 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -36,7 +36,7 @@ pub mod rt {
     use symbol::Symbol;
     use ThinVec;
 
-    use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
+    use tokenstream::{DelimSpan, TokenTree, TokenStream};
 
     pub use parse::new_parser_from_tts;
     pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
@@ -246,9 +246,9 @@ pub mod rt {
             inner.push(self.tokens.clone());
 
             let delim_span = DelimSpan::from_single(self.span);
-            r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
-                delim: token::Bracket, tts: TokenStream::concat(inner).into()
-            }));
+            r.push(TokenTree::Delimited(
+                delim_span, token::Bracket, TokenStream::concat(inner).into()
+            ));
             r
         }
     }
@@ -262,10 +262,9 @@ pub mod rt {
 
     impl ToTokens for () {
         fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
-            vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
-                delim: token::Paren,
-                tts: TokenStream::empty().into(),
-            })]
+            vec![
+                TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into())
+            ]
         }
     }
 
@@ -382,8 +381,6 @@ pub mod rt {
 
 // Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`.
 pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
-    use tokenstream::Delimited;
-
     let mut results = Vec::new();
     let mut result = Vec::new();
     let mut open_span = DUMMY_SP;
@@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
             }
             TokenTree::Token(span, token::CloseDelim(delim)) => {
                 let delim_span = DelimSpan::from_pair(open_span, span);
-                let tree = TokenTree::Delimited(delim_span, Delimited {
+                let tree = TokenTree::Delimited(
+                    delim_span,
                     delim,
-                    tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
-                });
+                    result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
+                );
                 result = results.pop().unwrap();
                 result.push(tree);
             }
@@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
                                     vec![e_tok]);
             vec![cx.stmt_expr(e_push)]
         },
-        TokenTree::Delimited(span, ref delimed) => {
-            let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
-            stmts.extend(statements_mk_tts(cx, delimed.stream()));
-            stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
+        TokenTree::Delimited(span, delim, ref tts) => {
+            let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false);
+            stmts.extend(statements_mk_tts(cx, tts.stream()));
+            stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false));
             stmts
         }
     }
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index 1dab0297854..b142f09cdbc 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -280,17 +280,17 @@ where
         // `tree` is a `$` token. Look at the next token in `trees`
         tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
             // `tree` is followed by a delimited set of token trees. This indicates the beginning
-            // of a repetition sequence in the macro (e.g., `$(pat)*`).
-            Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
+            // of a repetition sequence in the macro (e.g. `$(pat)*`).
+            Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
                 // Must have `(` not `{` or `[`
-                if delimited.delim != token::Paren {
-                    let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
+                if delim != token::Paren {
+                    let tok = pprust::token_to_string(&token::OpenDelim(delim));
                     let msg = format!("expected `(`, found `{}`", tok);
                     sess.span_diagnostic.span_err(span.entire(), &msg);
                 }
                 // Parse the contents of the sequence itself
                 let sequence = parse(
-                    delimited.tts.into(),
+                    tts.into(),
                     expect_matchers,
                     sess,
                     features,
@@ -354,12 +354,12 @@ where
 
         // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
         // descend into the delimited set and further parse it.
-        tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
+        tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
             span,
             Lrc::new(Delimited {
-                delim: delimited.delim,
+                delim: delim,
                 tts: parse(
-                    delimited.tts.into(),
+                    tts.into(),
                     expect_matchers,
                     sess,
                     features,
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 3d897d17e0b..a76779ffebd 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -17,7 +17,7 @@ use fold::noop_fold_tt;
 use parse::token::{self, Token, NtTT};
 use smallvec::SmallVec;
 use syntax_pos::DUMMY_SP;
-use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
+use tokenstream::{TokenStream, TokenTree, DelimSpan};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
@@ -105,10 +105,11 @@ pub fn transcribe(cx: &ExtCtxt,
                     if result_stack.is_empty() {
                         return TokenStream::concat(result);
                     }
-                    let tree = TokenTree::Delimited(span, Delimited {
-                        delim: forest.delim,
-                        tts: TokenStream::concat(result).into(),
-                    });
+                    let tree = TokenTree::Delimited(
+                        span,
+                        forest.delim,
+                        TokenStream::concat(result).into(),
+                    );
                     result = result_stack.pop().unwrap();
                     result.push(tree.into());
                 }