Rollup merge of #127233 - nnethercote:parser-cleanups, r=petrochenkov

Some parser cleanups

Cleanups I made while looking closely at this code.

r? ``@petrochenkov``

2 files changed, 26 insertions, 37 deletions

diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index 593c78df3cd..65f1b5dbaf5 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -204,12 +204,14 @@ impl Attribute {
 
     pub fn tokens(&self) -> TokenStream {
         match &self.kind {
-            AttrKind::Normal(normal) => normal
-                .tokens
-                .as_ref()
-                .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
-                .to_attr_token_stream()
-                .to_tokenstream(),
+            AttrKind::Normal(normal) => TokenStream::new(
+                normal
+                    .tokens
+                    .as_ref()
+                    .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
+                    .to_attr_token_stream()
+                    .to_token_trees(),
+            ),
             &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
                 token::DocComment(comment_kind, self.style, data),
                 self.span,
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index b4ddbe20689..655c18e4559 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -23,7 +23,6 @@ use rustc_data_structures::sync::{self, Lrc};
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::{sym, Span, SpanDecoder, SpanEncoder, Symbol, DUMMY_SP};
-use smallvec::{smallvec, SmallVec};
 
 use std::borrow::Cow;
 use std::{cmp, fmt, iter};
@@ -180,27 +179,25 @@ impl AttrTokenStream {
         AttrTokenStream(Lrc::new(tokens))
     }
 
-    /// Converts this `AttrTokenStream` to a plain `TokenStream`.
+    /// Converts this `AttrTokenStream` to a plain `Vec<TokenTree>`.
     /// During conversion, `AttrTokenTree::Attributes` get 'flattened'
     /// back to a `TokenStream` of the form `outer_attr attr_target`.
     /// If there are inner attributes, they are inserted into the proper
     /// place in the attribute target tokens.
-    pub fn to_tokenstream(&self) -> TokenStream {
-        let trees: Vec<_> = self
-            .0
-            .iter()
-            .flat_map(|tree| match &tree {
+    pub fn to_token_trees(&self) -> Vec<TokenTree> {
+        let mut res = Vec::with_capacity(self.0.len());
+        for tree in self.0.iter() {
+            match tree {
                 AttrTokenTree::Token(inner, spacing) => {
-                    smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
+                    res.push(TokenTree::Token(inner.clone(), *spacing));
                 }
                 AttrTokenTree::Delimited(span, spacing, delim, stream) => {
-                    smallvec![TokenTree::Delimited(
+                    res.push(TokenTree::Delimited(
                         *span,
                         *spacing,
                         *delim,
-                        stream.to_tokenstream()
-                    ),]
-                    .into_iter()
+                        TokenStream::new(stream.to_token_trees()),
+                    ))
                 }
                 AttrTokenTree::Attributes(data) => {
                     let idx = data
@@ -208,14 +205,7 @@ impl AttrTokenStream {
                         .partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
                     let (outer_attrs, inner_attrs) = data.attrs.split_at(idx);
 
-                    let mut target_tokens: Vec<_> = data
-                        .tokens
-                        .to_attr_token_stream()
-                        .to_tokenstream()
-                        .0
-                        .iter()
-                        .cloned()
-                        .collect();
+                    let mut target_tokens = data.tokens.to_attr_token_stream().to_token_trees();
                     if !inner_attrs.is_empty() {
                         let mut found = false;
                         // Check the last two trees (to account for a trailing semi)
@@ -251,17 +241,14 @@ impl AttrTokenStream {
                             "Failed to find trailing delimited group in: {target_tokens:?}"
                         );
                     }
-                    let mut flat: SmallVec<[_; 1]> =
-                        SmallVec::with_capacity(target_tokens.len() + outer_attrs.len());
                     for attr in outer_attrs {
-                        flat.extend(attr.tokens().0.iter().cloned());
+                        res.extend(attr.tokens().0.iter().cloned());
                     }
-                    flat.extend(target_tokens);
-                    flat.into_iter()
+                    res.extend(target_tokens);
                 }
-            })
-            .collect();
-        TokenStream::new(trees)
+            }
+        }
+        res
     }
 }
 
@@ -409,8 +396,8 @@ impl PartialEq<TokenStream> for TokenStream {
 }
 
 impl TokenStream {
-    pub fn new(streams: Vec<TokenTree>) -> TokenStream {
-        TokenStream(Lrc::new(streams))
+    pub fn new(tts: Vec<TokenTree>) -> TokenStream {
+        TokenStream(Lrc::new(tts))
     }
 
     pub fn is_empty(&self) -> bool {
@@ -461,7 +448,7 @@ impl TokenStream {
                 AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
             AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)])
         };
-        attr_stream.to_tokenstream()
+        TokenStream::new(attr_stream.to_token_trees())
     }
 
     pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {