Auto merge of #56369 - nnethercote:rm-Delimited, r=petrochenkov

Remove `tokenstream::Delimited`.

Because it's an extra type layer that doesn't really help; in a couple
of places it actively gets in the way, and overall removing it makes the
code nicer. It does, however, move `tokenstream::TokenTree` further away
from the `TokenTree` in `quote.rs`.

More importantly, this change reduces the size of `TokenStream` from 48
bytes to 40 bytes on x86-64, which is enough to slightly reduce
instruction counts on numerous benchmarks, the best by 1.5%.

Note that `open_tt` and `close_tt` have gone from being methods on
`Delimited` to associated methods of `TokenTree`.

4 files changed, 78 insertions, 79 deletions

diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
index 8047ab01465..86c87cf898d 100644
--- a/src/libsyntax/parse/lexer/tokentrees.rs
+++ b/src/libsyntax/parse/lexer/tokentrees.rs
@@ -11,7 +11,7 @@
 use print::pprust::token_to_string;
 use parse::lexer::StringReader;
 use parse::{token, PResult};
-use tokenstream::{Delimited, DelimSpan, TokenStream, TokenTree};
+use tokenstream::{DelimSpan, TokenStream, TokenTree};
 
 impl<'a> StringReader<'a> {
     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
@@ -155,10 +155,11 @@ impl<'a> StringReader<'a> {
                     _ => {}
                 }
 
-                Ok(TokenTree::Delimited(delim_span, Delimited {
+                Ok(TokenTree::Delimited(
+                    delim_span,
                     delim,
-                    tts: tts.into(),
-                }).into())
+                    tts.into(),
+                ).into())
             },
             token::CloseDelim(_) => {
                 // An unexpected closing delimiter (i.e., there is no
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 866dba24dcb..eb71003d3d0 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -764,7 +764,7 @@ mod tests {
     use attr::first_attr_value_str_by_name;
     use parse;
     use print::pprust::item_to_string;
-    use tokenstream::{self, DelimSpan, TokenTree};
+    use tokenstream::{DelimSpan, TokenTree};
     use util::parser_testing::string_to_stream;
     use util::parser_testing::{string_to_expr, string_to_item};
     use with_globals;
@@ -795,42 +795,41 @@ mod tests {
                     Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))),
                     Some(&TokenTree::Token(_, token::Not)),
                     Some(&TokenTree::Token(_, token::Ident(name_zip, false))),
-                    Some(&TokenTree::Delimited(_, ref macro_delimed)),
+                    Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)),
                 )
                 if name_macro_rules.name == "macro_rules"
                 && name_zip.name == "zip" => {
-                    let tts = &macro_delimed.stream().trees().collect::<Vec<_>>();
+                    let tts = &macro_tts.stream().trees().collect::<Vec<_>>();
                     match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) {
                         (
                             3,
-                            Some(&TokenTree::Delimited(_, ref first_delimed)),
+                            Some(&TokenTree::Delimited(_, first_delim, ref first_tts)),
                             Some(&TokenTree::Token(_, token::FatArrow)),
-                            Some(&TokenTree::Delimited(_, ref second_delimed)),
+                            Some(&TokenTree::Delimited(_, second_delim, ref second_tts)),
                         )
-                        if macro_delimed.delim == token::Paren => {
-                            let tts = &first_delimed.stream().trees().collect::<Vec<_>>();
+                        if macro_delim == token::Paren => {
+                            let tts = &first_tts.stream().trees().collect::<Vec<_>>();
                             match (tts.len(), tts.get(0), tts.get(1)) {
                                 (
                                     2,
                                     Some(&TokenTree::Token(_, token::Dollar)),
                                     Some(&TokenTree::Token(_, token::Ident(ident, false))),
                                 )
-                                if first_delimed.delim == token::Paren && ident.name == "a" => {},
-                                _ => panic!("value 3: {:?}", *first_delimed),
+                                if first_delim == token::Paren && ident.name == "a" => {},
+                                _ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
                             }
-                            let tts = &second_delimed.stream().trees().collect::<Vec<_>>();
+                            let tts = &second_tts.stream().trees().collect::<Vec<_>>();
                             match (tts.len(), tts.get(0), tts.get(1)) {
                                 (
                                     2,
                                     Some(&TokenTree::Token(_, token::Dollar)),
                                     Some(&TokenTree::Token(_, token::Ident(ident, false))),
                                 )
-                                if second_delimed.delim == token::Paren
-                                && ident.name == "a" => {},
-                                _ => panic!("value 4: {:?}", *second_delimed),
+                                if second_delim == token::Paren && ident.name == "a" => {},
+                                _ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
                             }
                         },
-                        _ => panic!("value 2: {:?}", *macro_delimed),
+                        _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
                     }
                 },
                 _ => panic!("value: {:?}",tts),
@@ -848,26 +847,24 @@ mod tests {
                 TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
-                    tokenstream::Delimited {
-                        delim: token::DelimToken::Paren,
-                        tts: TokenStream::concat(vec![
-                            TokenTree::Token(sp(6, 7),
-                                             token::Ident(Ident::from_str("b"), false)).into(),
-                            TokenTree::Token(sp(8, 9), token::Colon).into(),
-                            TokenTree::Token(sp(10, 13),
-                                             token::Ident(Ident::from_str("i32"), false)).into(),
-                        ]).into(),
-                    }).into(),
+                    token::DelimToken::Paren,
+                    TokenStream::concat(vec![
+                        TokenTree::Token(sp(6, 7),
+                                         token::Ident(Ident::from_str("b"), false)).into(),
+                        TokenTree::Token(sp(8, 9), token::Colon).into(),
+                        TokenTree::Token(sp(10, 13),
+                                         token::Ident(Ident::from_str("i32"), false)).into(),
+                    ]).into(),
+                ).into(),
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
-                    tokenstream::Delimited {
-                        delim: token::DelimToken::Brace,
-                        tts: TokenStream::concat(vec![
-                            TokenTree::Token(sp(17, 18),
-                                             token::Ident(Ident::from_str("b"), false)).into(),
-                            TokenTree::Token(sp(18, 19), token::Semi).into(),
-                        ]).into(),
-                    }).into()
+                    token::DelimToken::Brace,
+                    TokenStream::concat(vec![
+                        TokenTree::Token(sp(17, 18),
+                                         token::Ident(Ident::from_str("b"), false)).into(),
+                        TokenTree::Token(sp(18, 19), token::Semi).into(),
+                    ]).into(),
+                ).into()
             ]);
 
             assert_eq!(tts, expected);
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index e3a3454de49..ded6da9f3ad 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -48,13 +48,14 @@ use errors::{self, Applicability, DiagnosticBuilder, DiagnosticId};
 use parse::{self, SeqSep, classify, token};
 use parse::lexer::TokenAndSpan;
 use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
+use parse::token::DelimToken;
 use parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership};
 use util::parser::{AssocOp, Fixity};
 use print::pprust;
 use ptr::P;
 use parse::PResult;
 use ThinVec;
-use tokenstream::{self, Delimited, DelimSpan, ThinTokenStream, TokenTree, TokenStream};
+use tokenstream::{self, DelimSpan, ThinTokenStream, TokenTree, TokenStream};
 use symbol::{Symbol, keywords};
 
 use std::borrow::Cow;
@@ -293,13 +294,13 @@ enum LastToken {
 }
 
 impl TokenCursorFrame {
-    fn new(sp: DelimSpan, delimited: &Delimited) -> Self {
+    fn new(sp: DelimSpan, delim: DelimToken, tts: &ThinTokenStream) -> Self {
         TokenCursorFrame {
-            delim: delimited.delim,
+            delim: delim,
             span: sp,
-            open_delim: delimited.delim == token::NoDelim,
-            tree_cursor: delimited.stream().into_trees(),
-            close_delim: delimited.delim == token::NoDelim,
+            open_delim: delim == token::NoDelim,
+            tree_cursor: tts.stream().into_trees(),
+            close_delim: delim == token::NoDelim,
             last_token: LastToken::Was(None),
         }
     }
@@ -310,14 +311,12 @@ impl TokenCursor {
         loop {
             let tree = if !self.frame.open_delim {
                 self.frame.open_delim = true;
-                Delimited { delim: self.frame.delim, tts: TokenStream::empty().into() }
-                    .open_tt(self.frame.span.open)
+                TokenTree::open_tt(self.frame.span.open, self.frame.delim)
             } else if let Some(tree) = self.frame.tree_cursor.next() {
                 tree
             } else if !self.frame.close_delim {
                 self.frame.close_delim = true;
-                Delimited { delim: self.frame.delim, tts: TokenStream::empty().into() }
-                    .close_tt(self.frame.span.close)
+                TokenTree::close_tt(self.frame.span.close, self.frame.delim)
             } else if let Some(frame) = self.stack.pop() {
                 self.frame = frame;
                 continue
@@ -332,8 +331,8 @@ impl TokenCursor {
 
             match tree {
                 TokenTree::Token(sp, tok) => return TokenAndSpan { tok: tok, sp: sp },
-                TokenTree::Delimited(sp, ref delimited) => {
-                    let frame = TokenCursorFrame::new(sp, delimited);
+                TokenTree::Delimited(sp, delim, tts) => {
+                    let frame = TokenCursorFrame::new(sp, delim, &tts);
                     self.stack.push(mem::replace(&mut self.frame, frame));
                 }
             }
@@ -362,25 +361,28 @@ impl TokenCursor {
         }
 
         let delim_span = DelimSpan::from_single(sp);
-        let body = TokenTree::Delimited(delim_span, Delimited {
-            delim: token::Bracket,
-            tts: [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)),
-                  TokenTree::Token(sp, token::Eq),
-                  TokenTree::Token(sp, token::Literal(
-                      token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))]
-                .iter().cloned().collect::<TokenStream>().into(),
-        });
+        let body = TokenTree::Delimited(
+            delim_span,
+            token::Bracket,
+            [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)),
+             TokenTree::Token(sp, token::Eq),
+             TokenTree::Token(sp, token::Literal(
+                token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))
+            ]
+            .iter().cloned().collect::<TokenStream>().into(),
+        );
 
-        self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new(delim_span, &Delimited {
-            delim: token::NoDelim,
-            tts: if doc_comment_style(&name.as_str()) == AttrStyle::Inner {
+        self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new(
+            delim_span,
+            token::NoDelim,
+            &if doc_comment_style(&name.as_str()) == AttrStyle::Inner {
                 [TokenTree::Token(sp, token::Pound), TokenTree::Token(sp, token::Not), body]
                     .iter().cloned().collect::<TokenStream>().into()
             } else {
                 [TokenTree::Token(sp, token::Pound), body]
                     .iter().cloned().collect::<TokenStream>().into()
             },
-        })));
+        )));
 
         self.next()
     }
@@ -561,10 +563,11 @@ impl<'a> Parser<'a> {
             root_module_name: None,
             expected_tokens: Vec::new(),
             token_cursor: TokenCursor {
-                frame: TokenCursorFrame::new(DelimSpan::dummy(), &Delimited {
-                    delim: token::NoDelim,
-                    tts: tokens.into(),
-                }),
+                frame: TokenCursorFrame::new(
+                    DelimSpan::dummy(),
+                    token::NoDelim,
+                    &tokens.into(),
+                ),
                 stack: Vec::new(),
             },
             desugar_doc_comments,
@@ -1238,7 +1241,7 @@ impl<'a> Parser<'a> {
         f(&match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) {
             Some(tree) => match tree {
                 TokenTree::Token(_, tok) => tok,
-                TokenTree::Delimited(_, delimited) => token::OpenDelim(delimited.delim),
+                TokenTree::Delimited(_, delim, _) => token::OpenDelim(delim),
             },
             None => token::CloseDelim(self.token_cursor.frame.delim),
         })
@@ -1251,7 +1254,7 @@ impl<'a> Parser<'a> {
 
         match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) {
             Some(TokenTree::Token(span, _)) => span,
-            Some(TokenTree::Delimited(span, _)) => span.entire(),
+            Some(TokenTree::Delimited(span, ..)) => span.entire(),
             None => self.look_ahead_span(dist - 1),
         }
     }
@@ -2317,8 +2320,8 @@ impl<'a> Parser<'a> {
                 return Err(err)
             }
         };
-        let delimited = match self.parse_token_tree() {
-            TokenTree::Delimited(_, delimited) => delimited,
+        let tts = match self.parse_token_tree() {
+            TokenTree::Delimited(_, _, tts) => tts,
             _ => unreachable!(),
         };
         let delim = match delim {
@@ -2327,7 +2330,7 @@ impl<'a> Parser<'a> {
             token::Brace => MacDelimiter::Brace,
             token::NoDelim => self.bug("unexpected no delimiter"),
         };
-        Ok((delim, delimited.stream().into()))
+        Ok((delim, tts.stream().into()))
     }
 
     /// At the bottom (top?) of the precedence hierarchy,
@@ -2892,10 +2895,11 @@ impl<'a> Parser<'a> {
                                          self.token_cursor.stack.pop().unwrap());
                 self.span = frame.span.entire();
                 self.bump();
-                TokenTree::Delimited(frame.span, Delimited {
-                    delim: frame.delim,
-                    tts: frame.tree_cursor.original_stream().into(),
-                })
+                TokenTree::Delimited(
+                    frame.span,
+                    frame.delim,
+                    frame.tree_cursor.original_stream().into(),
+                )
             },
             token::CloseDelim(_) | token::Eof => unreachable!(),
             _ => {
@@ -4609,7 +4613,7 @@ impl<'a> Parser<'a> {
                 let ident = self.parse_ident()?;
                 let tokens = if self.check(&token::OpenDelim(token::Brace)) {
                     match self.parse_token_tree() {
-                        TokenTree::Delimited(_, ref delimited) => delimited.stream(),
+                        TokenTree::Delimited(_, _, tts) => tts.stream(),
                         _ => unreachable!(),
                     }
                 } else if self.check(&token::OpenDelim(token::Paren)) {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 6b9cc2f9792..8e4d3c0166b 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -818,16 +818,13 @@ fn prepend_attrs(sess: &ParseSess,
 
         brackets.push(attr.tokens.clone());
 
-        let tokens = tokenstream::Delimited {
-            delim: DelimToken::Bracket,
-            tts: brackets.build().into(),
-        };
         // The span we list here for `#` and for `[ ... ]` are both wrong in
         // that it encompasses more than each token, but it hopefully is "good
         // enough" for now at least.
         builder.push(tokenstream::TokenTree::Token(attr.span, Pound));
         let delim_span = DelimSpan::from_single(attr.span);
-        builder.push(tokenstream::TokenTree::Delimited(delim_span, tokens));
+        builder.push(tokenstream::TokenTree::Delimited(
+            delim_span, DelimToken::Bracket, brackets.build().into()));
     }
     builder.push(tokens.clone());
     Some(builder.build())