Rollup merge of #65455 - nnethercote:avoid-unnecessary-TokenTree-to-TokenStream-conversions, r=petrochenkov

Avoid unnecessary `TokenTree` to `TokenStream` conversions

A `TokenStream` contains any number of `TokenTrees`. Therefore, a single `TokenTree` can be promoted to a `TokenStream`. But doing so costs two allocations: one for the single-element `Vec`, and one for the `Lrc`. (An `IsJoint` value also must be added; the default is `NonJoint`.)

The current code converts `TokenTree`s to `TokenStream`s unnecessarily in a few places. This PR removes some of these unnecessary conversions, both simplifying the code and speeding it up.

r? @petrochenkov

3 files changed, 10 insertions, 11 deletions

diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs
index 14e1696610a..7952e293a53 100644
--- a/src/libsyntax/parse/literal.rs
+++ b/src/libsyntax/parse/literal.rs
@@ -3,7 +3,7 @@
 use crate::ast::{self, Lit, LitKind};
 use crate::parse::token::{self, Token};
 use crate::symbol::{kw, sym, Symbol};
-use crate::tokenstream::{TokenStream, TokenTree};
+use crate::tokenstream::TokenTree;
 
 use log::debug;
 use rustc_data_structures::sync::Lrc;
@@ -216,13 +216,13 @@ impl Lit {
         Lit { token: kind.to_lit_token(), kind, span }
     }
 
-    /// Losslessly convert an AST literal into a token stream.
-    crate fn tokens(&self) -> TokenStream {
+    /// Losslessly convert an AST literal into a token tree.
+    crate fn token_tree(&self) -> TokenTree {
         let token = match self.token.kind {
             token::Bool => token::Ident(self.token.symbol, false),
             _ => token::Literal(self.token),
         };
-        TokenTree::token(token, self.span).into()
+        TokenTree::token(token, self.span)
     }
 }
 
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 9cb410a8ae3..6bbd8be0cb9 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -285,10 +285,10 @@ impl TokenCursor {
             token::NoDelim,
             &if doc_comment_style(&name.as_str()) == AttrStyle::Inner {
                 [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body]
-                    .iter().cloned().collect::<TokenStream>().into()
+                    .iter().cloned().collect::<TokenStream>()
             } else {
                 [TokenTree::token(token::Pound, sp), body]
-                    .iter().cloned().collect::<TokenStream>().into()
+                    .iter().cloned().collect::<TokenStream>()
             },
         )));
 
diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs
index 6f7d1ead4c1..188a144cac9 100644
--- a/src/libsyntax/parse/parser/attr.rs
+++ b/src/libsyntax/parse/parser/attr.rs
@@ -6,7 +6,6 @@ use crate::tokenstream::{TokenStream, TokenTree};
 use crate::source_map::Span;
 
 use log::debug;
-use smallvec::smallvec;
 
 #[derive(Debug)]
 enum InnerAttributeParsePolicy<'a> {
@@ -193,15 +192,15 @@ impl<'a> Parser<'a> {
                         is_interpolated_expr = true;
                     }
                 }
-                let tokens = if is_interpolated_expr {
+                let token_tree = if is_interpolated_expr {
                     // We need to accept arbitrary interpolated expressions to continue
                     // supporting things like `doc = $expr` that work on stable.
                     // Non-literal interpolated expressions are rejected after expansion.
-                    self.parse_token_tree().into()
+                    self.parse_token_tree()
                 } else {
-                    self.parse_unsuffixed_lit()?.tokens()
+                    self.parse_unsuffixed_lit()?.token_tree()
                 };
-                TokenStream::from_streams(smallvec![eq.into(), tokens])
+                TokenStream::new(vec![eq.into(), token_tree.into()])
             } else {
                 TokenStream::default()
             };