Rollup merge of #65641 - nnethercote:derive-TokenStream-Encodable-Decodable, r=petrochenkov

Derive `Rustc{En,De}codable` for `TokenStream`.

`TokenStream` used to be a complex type, but it is now just a newtype
around a `Lrc<Vec<TreeAndJoint>>`. Currently it uses custom encoding
that discards the `IsJoint` and custom decoding that adds `NonJoint`
back in for every token tree. This requires building intermediate
`Vec<TokenTree>`s.

This commit makes `TokenStream` derive `Rustc{En,De}codable`. This
simplifies the code, and avoids the creation of the intermediate
vectors, saving up to 3% on various benchmarks. It also changes the AST
JSON output in one test.

r? @petrochenkov

1 files changed, 2 insertions, 15 deletions

diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index ac155556cda..0559f224f1f 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -19,7 +19,6 @@ use syntax_pos::{BytePos, Span, DUMMY_SP};
 #[cfg(target_arch = "x86_64")]
 use rustc_data_structures::static_assert_size;
 use rustc_data_structures::sync::Lrc;
-use rustc_serialize::{Decoder, Decodable, Encoder, Encodable};
 use smallvec::{SmallVec, smallvec};
 
 use std::{iter, mem};
@@ -136,7 +135,7 @@ impl TokenTree {
 /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
 /// instead of a representation of the abstract syntax tree.
 /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, RustcEncodable, RustcDecodable)]
 pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>);
 
 pub type TreeAndJoint = (TokenTree, IsJoint);
@@ -145,7 +144,7 @@ pub type TreeAndJoint = (TokenTree, IsJoint);
 #[cfg(target_arch = "x86_64")]
 static_assert_size!(TokenStream, 8);
 
-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq, RustcEncodable, RustcDecodable)]
 pub enum IsJoint {
     Joint,
     NonJoint
@@ -460,18 +459,6 @@ impl Cursor {
     }
 }
 
-impl Encodable for TokenStream {
-    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), E::Error> {
-        self.trees().collect::<Vec<_>>().encode(encoder)
-    }
-}
-
-impl Decodable for TokenStream {
-    fn decode<D: Decoder>(decoder: &mut D) -> Result<TokenStream, D::Error> {
-        Vec::<TokenTree>::decode(decoder).map(|vec| vec.into_iter().collect())
-    }
-}
-
 #[derive(Debug, Copy, Clone, PartialEq, RustcEncodable, RustcDecodable)]
 pub struct DelimSpan {
     pub open: Span,