Rollup merge of #39118 - jseyfried:token_tree_based_parser, r=nrc

Refactor the parser to consume token trees

This is groundwork for efficiently parsing attribute proc macro invocations, bang macro invocations, and `TokenStream`-based attributes and fragment matchers.

This improves parsing performance by 8-15% and expansion performance by 0-5% on a sampling of the compiler's crates.

r? @nrc

1 files changed, 6 insertions, 7 deletions

diff --git a/src/librustc_save_analysis/span_utils.rs b/src/librustc_save_analysis/span_utils.rs
index 448bb2e7617..89525b27ed3 100644
--- a/src/librustc_save_analysis/span_utils.rs
+++ b/src/librustc_save_analysis/span_utils.rs
@@ -17,9 +17,9 @@ use std::env;
 use std::path::Path;
 
 use syntax::ast;
-use syntax::parse::lexer::{self, Reader, StringReader};
+use syntax::parse::filemap_to_tts;
+use syntax::parse::lexer::{self, StringReader};
 use syntax::parse::token::{self, Token};
-use syntax::parse::parser::Parser;
 use syntax::symbol::keywords;
 use syntax::tokenstream::TokenTree;
 use syntax_pos::*;
@@ -85,14 +85,13 @@ impl<'a> SpanUtils<'a> {
         let filemap = self.sess
                           .codemap()
                           .new_filemap(String::from("<anon-dxr>"), None, self.snippet(span));
-        let s = self.sess;
-        lexer::StringReader::new(s.diagnostic(), filemap)
+        lexer::StringReader::new(&self.sess.parse_sess, filemap)
     }
 
     fn span_to_tts(&self, span: Span) -> Vec<TokenTree> {
-        let srdr = self.retokenise_span(span);
-        let mut p = Parser::new(&self.sess.parse_sess, Box::new(srdr), None, false);
-        p.parse_all_token_trees().expect("Couldn't re-parse span")
+        let filename = String::from("<anon-dxr>");
+        let filemap = self.sess.codemap().new_filemap(filename, None, self.snippet(span));
+        filemap_to_tts(&self.sess.parse_sess, filemap)
     }
 
     // Re-parses a path and returns the span for the last identifier in the path