Auto merge of #39173 - jseyfried:tokenstream, r=nrc

Refactor `TokenStream`

r? @nrc

4 files changed, 10 insertions, 19 deletions

diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
index 7b6f00e0e82..eafc3f77ab0 100644
--- a/src/libsyntax/parse/lexer/tokentrees.rs
+++ b/src/libsyntax/parse/lexer/tokentrees.rs
@@ -59,7 +59,6 @@ impl<'a> StringReader<'a> {
 
                 // Parse the open delimiter.
                 self.open_braces.push((delim, self.span));
-                let open_span = self.span;
                 self.real_token();
 
                 // Parse the token trees within the delimiters.
@@ -67,9 +66,8 @@ impl<'a> StringReader<'a> {
                 // uses an incorrect delimiter.
                 let tts = self.parse_token_trees_until_close_delim();
 
-                let close_span = self.span;
                 // Expand to cover the entire delimited token tree
-                let span = Span { hi: close_span.hi, ..pre_span };
+                let span = Span { hi: self.span.hi, ..pre_span };
 
                 match self.token {
                     // Correct delimiter.
@@ -115,9 +113,7 @@ impl<'a> StringReader<'a> {
 
                 Ok(TokenTree::Delimited(span, Rc::new(Delimited {
                     delim: delim,
-                    open_span: open_span,
                     tts: tts,
-                    close_span: close_span,
                 })))
             },
             token::CloseDelim(_) => {
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 08f5df4515b..f1a3b523cfd 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -196,7 +196,7 @@ pub fn new_parser_from_tts<'a>(sess: &'a ParseSess, tts: Vec<tokenstream::TokenT
 }
 
 pub fn new_parser_from_ts<'a>(sess: &'a ParseSess, ts: tokenstream::TokenStream) -> Parser<'a> {
-    tts_to_parser(sess, ts.to_tts())
+    tts_to_parser(sess, ts.trees().cloned().collect())
 }
 
 
@@ -725,24 +725,20 @@ mod tests {
                 sp(5, 14),
                 Rc::new(tokenstream::Delimited {
                     delim: token::DelimToken::Paren,
-                    open_span: sp(5, 6),
                     tts: vec![
                         TokenTree::Token(sp(6, 7), token::Ident(Ident::from_str("b"))),
                         TokenTree::Token(sp(8, 9), token::Colon),
                         TokenTree::Token(sp(10, 13), token::Ident(Ident::from_str("i32"))),
                     ],
-                    close_span: sp(13, 14),
                 })),
             TokenTree::Delimited(
                 sp(15, 21),
                 Rc::new(tokenstream::Delimited {
                     delim: token::DelimToken::Brace,
-                    open_span: sp(15, 16),
                     tts: vec![
                         TokenTree::Token(sp(17, 18), token::Ident(Ident::from_str("b"))),
                         TokenTree::Token(sp(18, 19), token::Semi),
                     ],
-                    close_span: sp(20, 21),
                 }))
         ];
 
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 766e0512bdc..2a0a7385998 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -256,9 +256,7 @@ impl<'a> Parser<'a> {
                -> Self {
         let tt = TokenTree::Delimited(syntax_pos::DUMMY_SP, Rc::new(Delimited {
             delim: token::NoDelim,
-            open_span: syntax_pos::DUMMY_SP,
             tts: tokens,
-            close_span: syntax_pos::DUMMY_SP,
         }));
         let mut parser = Parser {
             sess: sess,
@@ -2720,7 +2718,7 @@ impl<'a> Parser<'a> {
                 }
 
                 let parsing_token_tree = ::std::mem::replace(&mut self.parsing_token_tree, true);
-                let open_span = self.span;
+                let lo = self.span.lo;
                 self.bump();
                 let tts = self.parse_seq_to_before_tokens(&[&token::CloseDelim(token::Brace),
                                                             &token::CloseDelim(token::Paren),
@@ -2729,16 +2727,11 @@ impl<'a> Parser<'a> {
                                                           |p| p.parse_token_tree(),
                                                           |mut e| e.emit());
                 self.parsing_token_tree = parsing_token_tree;
-
-                let close_span = self.span;
                 self.bump();
 
-                let span = Span { lo: open_span.lo, ..close_span };
-                Ok(TokenTree::Delimited(span, Rc::new(Delimited {
+                Ok(TokenTree::Delimited(Span { lo: lo, ..self.prev_span }, Rc::new(Delimited {
                     delim: delim,
-                    open_span: open_span,
                     tts: tts,
-                    close_span: close_span,
                 })))
             },
             token::CloseDelim(_) | token::Eof => unreachable!(),
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 8ac39dd462e..bf790b96e37 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -49,6 +49,12 @@ pub enum DelimToken {
     NoDelim,
 }
 
+impl DelimToken {
+    pub fn len(&self) -> u32 {
+        if *self == NoDelim { 0 } else { 1 }
+    }
+}
+
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
 pub enum Lit {
     Byte(ast::Name),