From e80a93040ffbbb7eb8013f1dcd3b594ce8a631cd Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 19 Dec 2018 14:53:52 +1100 Subject: Make `TokenStream` less recursive. `TokenStream` is currently recursive in *two* ways: - the `TokenTree` variant contains a `ThinTokenStream`, which can contain a `TokenStream`; - the `TokenStream` variant contains a `Vec`. The latter is not necessary and causes significant complexity. This commit replaces it with the simpler `Vec<(TokenTree, IsJoint)>`. This reduces complexity significantly. In particular, `StreamCursor` is eliminated, and `Cursor` becomes much simpler, consisting now of just a `TokenStream` and an index. The commit also removes the `Extend` impl for `TokenStream`, because it is only used in tests. (The commit also removes those tests.) Overall, the commit reduces the number of lines of code by almost 200. --- src/libsyntax/attr/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/libsyntax/attr') diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index a309775a1a4..d03563f8891 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -472,7 +472,7 @@ impl MetaItem { Token::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } - idents.push(self.node.tokens(self.span)); + self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); TokenStream::new(idents) } @@ -529,7 +529,9 @@ impl MetaItemKind { match *self { MetaItemKind::Word => TokenStream::empty(), MetaItemKind::NameValue(ref lit) => { - TokenStream::new(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()]) + let mut vec = vec![TokenTree::Token(span, Token::Eq).into()]; + lit.tokens().append_to_tree_and_joint_vec(&mut vec); + TokenStream::new(vec) } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -537,7 +539,7 @@ impl MetaItemKind { if i > 0 { tokens.push(TokenTree::Token(span, Token::Comma).into()); } - tokens.push(item.node.tokens()); + item.node.tokens().append_to_tree_and_joint_vec(&mut tokens); } TokenTree::Delimited( DelimSpan::from_single(span), -- cgit 1.4.1-3-g733a5