Rollup merge of #58001 - pnkfelix:issue-57735-proc-macro-with-large-tokenstream-slow, r=eddyb

proc_macro: make `TokenStream::from_streams` pre-allocate its vector.

This requires a pre-pass over the input streams. But that is cheap compared to the quadratic blowup associated with reallocating the accumulating vector on-the-fly.

Fix #57735

1 files changed, 7 insertions, 1 deletions

diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index f5d2d6f18ee..7d6ffceb2c0 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -255,7 +255,13 @@ impl TokenStream {
             0 => TokenStream::empty(),
             1 => streams.pop().unwrap(),
             _ => {
-                let mut vec = vec![];
+                // rust-lang/rust#57735: pre-allocate vector to avoid
+                // quadratic blow-up due to on-the-fly reallocations.
+                let tree_count = streams.iter()
+                    .map(|ts| match &ts.0 { None => 0, Some(s) => s.len() })
+                    .sum();
+                let mut vec = Vec::with_capacity(tree_count);
+
                 for stream in streams {
                     match stream.0 {
                         None => {},