From e80a93040ffbbb7eb8013f1dcd3b594ce8a631cd Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 19 Dec 2018 14:53:52 +1100 Subject: Make `TokenStream` less recursive. `TokenStream` is currently recursive in *two* ways: - the `TokenTree` variant contains a `ThinTokenStream`, which can contain a `TokenStream`; - the `TokenStream` variant contains a `Vec`. The latter is not necessary and causes significant complexity. This commit replaces it with the simpler `Vec<(TokenTree, IsJoint)>`. This reduces complexity significantly. In particular, `StreamCursor` is eliminated, and `Cursor` becomes much simpler, consisting now of just a `TokenStream` and an index. The commit also removes the `Extend` impl for `TokenStream`, because it is only used in tests. (The commit also removes those tests.) Overall, the commit reduces the number of lines of code by almost 200. --- src/libsyntax/ext/quote.rs | 2 +- src/libsyntax/ext/tt/transcribe.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/libsyntax/ext') diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index ad8668aca70..c3124144009 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -233,7 +233,7 @@ pub mod rt { self.span, token::Token::from_ast_ident(segment.ident) ).into()); } - inner.push(self.tokens.clone()); + self.tokens.clone().append_to_tree_and_joint_vec(&mut inner); let delim_span = DelimSpan::from_single(self.span); r.push(TokenTree::Delimited( diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 31d87508c6b..0ef2d3b749d 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -7,7 +7,7 @@ use fold::noop_fold_tt; use parse::token::{self, Token, NtTT}; use smallvec::SmallVec; use syntax_pos::DUMMY_SP; -use tokenstream::{TokenStream, TokenTree, DelimSpan}; +use tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -63,7 +63,7 @@ pub fn transcribe(cx: &ExtCtxt, let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; let interpolations = interp.unwrap_or_else(FxHashMap::default); /* just a convenience */ let mut repeats = Vec::new(); - let mut result: Vec = Vec::new(); + let mut result: Vec = Vec::new(); let mut result_stack = Vec::new(); loop { @@ -78,7 +78,7 @@ pub fn transcribe(cx: &ExtCtxt, if let Some(sep) = sep.clone() { // repeat same span, I guess let prev_span = match result.last() { - Some(stream) => stream.trees().next().unwrap().span(), + Some((tt, _)) => tt.span(), None => DUMMY_SP, }; result.push(TokenTree::Token(prev_span, sep).into()); -- cgit 1.4.1-3-g733a5